This model was created with the goal for a good llama 3 uncencored model with long context. At it worked like a charm.
70B
1,117 Pulls Updated 5 weeks ago
9fa2546ac1b9 · 58GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 18,
"general.name": "Llama-3-70b-Uncensored-Lumi-Tess-gradient",
"general.quantization_version": 2,
"general.source.url": "https://huggingface.co/ryzen88/Llama-3-70b-Uncensored-Lumi-Tess-gradient",
"general.url": "https://huggingface.co/mradermacher/Llama-3-70b-Uncensored-Lumi-Tess-gradient-GGUF",
"llama.attention.head_count": 64,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 80,
"llama.context_length": 262144,
"llama.embedding_length": 8192,
"llama.feed_forward_length": 28672,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 207112200,
"llama.vocab_size": 128256,
"mradermacher.convert_type": "hf",
"mradermacher.quantize_version": "2",
"mradermacher.quantized_at": "2024-05-15T07:00:29+02:00",
"mradermacher.quantized_by": "mradermacher",
"mradermacher.quantized_on": "db1",
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.pre": "llama-bpe",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 70553706496,
"tensors": [
{
"name": "token_embd.weight",
"offset": 861880320,
"shape": [
8192,
128256
],
"size": 861880320,
"type": 14
},
{
"name": "blk.0.attn_norm.weight",
"offset": 1723760640,
"shape": [
8192
],
"size": 861880320,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 1723793408,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 1916469248,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.0.ffn_up.weight",
"offset": 2109145088,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 2301820928,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 2301853696,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.0.attn_output.weight",
"offset": 2308734976,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.0.attn_q.weight",
"offset": 2363785216,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.0.attn_v.weight",
"offset": 2418835456,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 2425716736,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 2425749504,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 2618425344,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.1.ffn_up.weight",
"offset": 2811101184,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 3003777024,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 3003809792,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.1.attn_output.weight",
"offset": 3010691072,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.1.attn_q.weight",
"offset": 3065741312,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.1.attn_v.weight",
"offset": 3120791552,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.10.attn_norm.weight",
"offset": 3127672832,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 3127705600,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 3320381440,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.10.ffn_up.weight",
"offset": 3513057280,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 3705733120,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 3705765888,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.10.attn_output.weight",
"offset": 3712647168,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.10.attn_q.weight",
"offset": 3767697408,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.10.attn_v.weight",
"offset": 3822747648,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 3829628928,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 3829661696,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 4022337536,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.11.ffn_up.weight",
"offset": 4215013376,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 4407689216,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 4407721984,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.11.attn_output.weight",
"offset": 4414603264,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.11.attn_q.weight",
"offset": 4469653504,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.11.attn_v.weight",
"offset": 4524703744,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 4531585024,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 4531617792,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 4724293632,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.12.ffn_up.weight",
"offset": 4916969472,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 5109645312,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 5109678080,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.12.attn_output.weight",
"offset": 5116559360,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.12.attn_q.weight",
"offset": 5171609600,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.12.attn_v.weight",
"offset": 5226659840,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.13.attn_norm.weight",
"offset": 5233541120,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 5233573888,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 5426249728,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.13.ffn_up.weight",
"offset": 5618925568,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 5811601408,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 5811634176,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.13.attn_output.weight",
"offset": 5818515456,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.13.attn_q.weight",
"offset": 5873565696,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.13.attn_v.weight",
"offset": 5928615936,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 5935497216,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 5935529984,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 6128205824,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.14.ffn_up.weight",
"offset": 6320881664,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 6513557504,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 6513590272,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.14.attn_output.weight",
"offset": 6520471552,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.14.attn_q.weight",
"offset": 6575521792,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.14.attn_v.weight",
"offset": 6630572032,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 6637453312,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 6637486080,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 6830161920,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.15.ffn_up.weight",
"offset": 7022837760,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 7215513600,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 7215546368,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.15.attn_output.weight",
"offset": 7222427648,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.15.attn_q.weight",
"offset": 7277477888,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.15.attn_v.weight",
"offset": 7332528128,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.16.attn_norm.weight",
"offset": 7339409408,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 7339442176,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 7532118016,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.16.ffn_up.weight",
"offset": 7724793856,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 7917469696,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 7917502464,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.16.attn_output.weight",
"offset": 7924383744,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.16.attn_q.weight",
"offset": 7979433984,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.16.attn_v.weight",
"offset": 8034484224,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 8041365504,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 8041398272,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 8234074112,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.17.ffn_up.weight",
"offset": 8426749952,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 8619425792,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 8619458560,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.17.attn_output.weight",
"offset": 8626339840,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.17.attn_q.weight",
"offset": 8681390080,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.17.attn_v.weight",
"offset": 8736440320,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 8743321600,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 8743354368,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 8936030208,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.18.ffn_up.weight",
"offset": 9128706048,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 9321381888,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 9321414656,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.18.attn_output.weight",
"offset": 9328295936,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.18.attn_q.weight",
"offset": 9383346176,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.18.attn_v.weight",
"offset": 9438396416,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.19.attn_norm.weight",
"offset": 9445277696,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 9445310464,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 9637986304,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.19.ffn_up.weight",
"offset": 9830662144,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 10023337984,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 10023370752,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.19.attn_output.weight",
"offset": 10030252032,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.19.attn_q.weight",
"offset": 10085302272,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.19.attn_v.weight",
"offset": 10140352512,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 10147233792,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 10147266560,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 10339942400,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.2.ffn_up.weight",
"offset": 10532618240,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 10725294080,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 10725326848,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.2.attn_output.weight",
"offset": 10732208128,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.2.attn_q.weight",
"offset": 10787258368,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.2.attn_v.weight",
"offset": 10842308608,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 10849189888,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 10849222656,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 11041898496,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.20.ffn_up.weight",
"offset": 11234574336,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 11427250176,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 11427282944,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.20.attn_output.weight",
"offset": 11434164224,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.20.attn_q.weight",
"offset": 11489214464,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.20.attn_v.weight",
"offset": 11544264704,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.21.attn_norm.weight",
"offset": 11551145984,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 11551178752,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 11743854592,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.21.ffn_up.weight",
"offset": 11936530432,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 12129206272,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 12129239040,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.21.attn_output.weight",
"offset": 12136120320,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.21.attn_q.weight",
"offset": 12191170560,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.21.attn_v.weight",
"offset": 12246220800,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 12253102080,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 12253134848,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 12445810688,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.22.ffn_up.weight",
"offset": 12638486528,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 12831162368,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 12831195136,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.22.attn_output.weight",
"offset": 12838076416,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.22.attn_q.weight",
"offset": 12893126656,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.22.attn_v.weight",
"offset": 12948176896,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 12955058176,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 12955090944,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 13147766784,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.23.ffn_up.weight",
"offset": 13340442624,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 13533118464,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 13533151232,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.23.attn_output.weight",
"offset": 13540032512,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.23.attn_q.weight",
"offset": 13595082752,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.23.attn_v.weight",
"offset": 13650132992,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.24.attn_norm.weight",
"offset": 13657014272,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 13657047040,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 13849722880,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.24.ffn_up.weight",
"offset": 14042398720,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 14235074560,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 14235107328,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.24.attn_output.weight",
"offset": 14241988608,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.24.attn_q.weight",
"offset": 14297038848,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.24.attn_v.weight",
"offset": 14352089088,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 14358970368,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 14359003136,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 14551678976,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.25.ffn_up.weight",
"offset": 14744354816,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 14937030656,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 14937063424,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.25.attn_output.weight",
"offset": 14943944704,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.25.attn_q.weight",
"offset": 14998994944,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.25.attn_v.weight",
"offset": 15054045184,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.26.attn_norm.weight",
"offset": 15060926464,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 15060959232,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 15253635072,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.26.ffn_up.weight",
"offset": 15446310912,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 15638986752,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 15639019520,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.26.attn_output.weight",
"offset": 15645900800,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.26.attn_q.weight",
"offset": 15700951040,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.26.attn_v.weight",
"offset": 15756001280,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.27.attn_norm.weight",
"offset": 15762882560,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 15762915328,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 15955591168,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.27.ffn_up.weight",
"offset": 16148267008,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 16340942848,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 16340975616,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.27.attn_output.weight",
"offset": 16347856896,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.27.attn_q.weight",
"offset": 16402907136,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.27.attn_v.weight",
"offset": 16457957376,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 16464838656,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 16464871424,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 16657547264,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.28.ffn_up.weight",
"offset": 16850223104,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 17042898944,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 17042931712,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.28.attn_output.weight",
"offset": 17049812992,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.28.attn_q.weight",
"offset": 17104863232,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.28.attn_v.weight",
"offset": 17159913472,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 17166794752,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 17166827520,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 17359503360,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.29.ffn_up.weight",
"offset": 17552179200,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 17744855040,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 17744887808,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.29.attn_output.weight",
"offset": 17751769088,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.29.attn_q.weight",
"offset": 17806819328,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.29.attn_v.weight",
"offset": 17861869568,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 17868750848,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 17868783616,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 18061459456,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.3.ffn_up.weight",
"offset": 18254135296,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 18446811136,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 18446843904,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.3.attn_output.weight",
"offset": 18453725184,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.3.attn_q.weight",
"offset": 18508775424,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.3.attn_v.weight",
"offset": 18563825664,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 18570706944,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 18570739712,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 18763415552,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.30.ffn_up.weight",
"offset": 18956091392,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 19148767232,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 19148800000,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.30.attn_output.weight",
"offset": 19155681280,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.30.attn_q.weight",
"offset": 19210731520,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.30.attn_v.weight",
"offset": 19265781760,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 19272663040,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 19272695808,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 19465371648,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.31.ffn_up.weight",
"offset": 19658047488,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 19850723328,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 19850756096,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.31.attn_output.weight",
"offset": 19857637376,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.31.attn_q.weight",
"offset": 19912687616,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.31.attn_v.weight",
"offset": 19967737856,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.32.attn_norm.weight",
"offset": 19974619136,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 19974651904,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 20167327744,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.32.ffn_up.weight",
"offset": 20360003584,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 20552679424,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 20552712192,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.32.attn_output.weight",
"offset": 20559593472,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.32.attn_q.weight",
"offset": 20614643712,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.32.attn_v.weight",
"offset": 20669693952,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.33.attn_norm.weight",
"offset": 20676575232,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 20676608000,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 20869283840,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.33.ffn_up.weight",
"offset": 21061959680,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 21254635520,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.33.attn_k.weight",
"offset": 21254668288,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.33.attn_output.weight",
"offset": 21261549568,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.33.attn_q.weight",
"offset": 21316599808,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.33.attn_v.weight",
"offset": 21371650048,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.34.attn_norm.weight",
"offset": 21378531328,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 21378564096,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 21571239936,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.34.ffn_up.weight",
"offset": 21763915776,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 21956591616,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 21956624384,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.34.attn_output.weight",
"offset": 21963505664,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.34.attn_q.weight",
"offset": 22018555904,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.34.attn_v.weight",
"offset": 22073606144,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.35.attn_norm.weight",
"offset": 22080487424,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 22080520192,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 22273196032,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.35.ffn_up.weight",
"offset": 22465871872,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 22658547712,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 22658580480,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.35.attn_output.weight",
"offset": 22665461760,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.35.attn_q.weight",
"offset": 22720512000,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.35.attn_v.weight",
"offset": 22775562240,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.36.attn_norm.weight",
"offset": 22782443520,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 22782476288,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 22975152128,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.36.ffn_up.weight",
"offset": 23167827968,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 23360503808,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 23360536576,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.36.attn_output.weight",
"offset": 23367417856,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.36.attn_q.weight",
"offset": 23422468096,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.36.attn_v.weight",
"offset": 23477518336,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.37.attn_norm.weight",
"offset": 23484399616,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 23484432384,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 23677108224,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.37.ffn_up.weight",
"offset": 23869784064,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 24062459904,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 24062492672,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.37.attn_output.weight",
"offset": 24069373952,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.37.attn_q.weight",
"offset": 24124424192,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.37.attn_v.weight",
"offset": 24179474432,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.38.attn_norm.weight",
"offset": 24186355712,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 24186388480,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 24379064320,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.38.ffn_up.weight",
"offset": 24571740160,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 24764416000,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.38.attn_k.weight",
"offset": 24764448768,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.38.attn_output.weight",
"offset": 24771330048,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.38.attn_q.weight",
"offset": 24826380288,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.38.attn_v.weight",
"offset": 24881430528,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.39.attn_norm.weight",
"offset": 24888311808,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 24888344576,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 25081020416,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.39.ffn_up.weight",
"offset": 25273696256,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 25466372096,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 25466404864,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.39.attn_output.weight",
"offset": 25473286144,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.39.attn_q.weight",
"offset": 25528336384,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.39.attn_v.weight",
"offset": 25583386624,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 25590267904,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 25590300672,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 25782976512,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.4.ffn_up.weight",
"offset": 25975652352,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 26168328192,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 26168360960,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.4.attn_output.weight",
"offset": 26175242240,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.4.attn_q.weight",
"offset": 26230292480,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.4.attn_v.weight",
"offset": 26285342720,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.40.attn_norm.weight",
"offset": 26292224000,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.40.ffn_down.weight",
"offset": 26292256768,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 26484932608,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.40.ffn_up.weight",
"offset": 26677608448,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 26870284288,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 26870317056,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.40.attn_output.weight",
"offset": 26877198336,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.40.attn_q.weight",
"offset": 26932248576,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.40.attn_v.weight",
"offset": 26987298816,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.41.attn_norm.weight",
"offset": 26994180096,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 26994212864,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 27186888704,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.41.ffn_up.weight",
"offset": 27379564544,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 27572240384,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.41.attn_k.weight",
"offset": 27572273152,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.41.attn_output.weight",
"offset": 27579154432,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.41.attn_q.weight",
"offset": 27634204672,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.41.attn_v.weight",
"offset": 27689254912,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.42.attn_norm.weight",
"offset": 27696136192,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 27696168960,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 27888844800,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.42.ffn_up.weight",
"offset": 28081520640,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 28274196480,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.42.attn_k.weight",
"offset": 28274229248,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.42.attn_output.weight",
"offset": 28281110528,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.42.attn_q.weight",
"offset": 28336160768,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.42.attn_v.weight",
"offset": 28391211008,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.43.attn_norm.weight",
"offset": 28398092288,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 28398125056,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 28590800896,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.43.ffn_up.weight",
"offset": 28783476736,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 28976152576,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 28976185344,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.43.attn_output.weight",
"offset": 28983066624,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.43.attn_q.weight",
"offset": 29038116864,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.43.attn_v.weight",
"offset": 29093167104,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.44.attn_norm.weight",
"offset": 29100048384,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 29100081152,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 29292756992,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.44.ffn_up.weight",
"offset": 29485432832,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 29678108672,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 29678141440,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.44.attn_output.weight",
"offset": 29685022720,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.44.attn_q.weight",
"offset": 29740072960,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.44.attn_v.weight",
"offset": 29795123200,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.45.attn_norm.weight",
"offset": 29802004480,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 29802037248,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 29994713088,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.45.ffn_up.weight",
"offset": 30187388928,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 30380064768,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 30380097536,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.45.attn_output.weight",
"offset": 30386978816,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.45.attn_q.weight",
"offset": 30442029056,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.45.attn_v.weight",
"offset": 30497079296,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.46.attn_norm.weight",
"offset": 30503960576,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 30503993344,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 30696669184,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.46.ffn_up.weight",
"offset": 30889345024,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 31082020864,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.46.attn_k.weight",
"offset": 31082053632,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.46.attn_output.weight",
"offset": 31088934912,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.46.attn_q.weight",
"offset": 31143985152,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.46.attn_v.weight",
"offset": 31199035392,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.47.attn_norm.weight",
"offset": 31205916672,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 31205949440,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 31398625280,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.47.ffn_up.weight",
"offset": 31591301120,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 31783976960,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.47.attn_k.weight",
"offset": 31784009728,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.47.attn_output.weight",
"offset": 31790891008,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.47.attn_q.weight",
"offset": 31845941248,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.47.attn_v.weight",
"offset": 31900991488,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.48.attn_norm.weight",
"offset": 31907872768,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.48.ffn_down.weight",
"offset": 31907905536,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 32100581376,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.48.ffn_up.weight",
"offset": 32293257216,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 32485933056,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.48.attn_k.weight",
"offset": 32485965824,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.48.attn_output.weight",
"offset": 32492847104,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.48.attn_q.weight",
"offset": 32547897344,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.48.attn_v.weight",
"offset": 32602947584,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.49.attn_norm.weight",
"offset": 32609828864,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.49.ffn_down.weight",
"offset": 32609861632,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 32802537472,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.49.ffn_up.weight",
"offset": 32995213312,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 33187889152,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.49.attn_k.weight",
"offset": 33187921920,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.49.attn_output.weight",
"offset": 33194803200,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.49.attn_q.weight",
"offset": 33249853440,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.49.attn_v.weight",
"offset": 33304903680,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.5.attn_norm.weight",
"offset": 33311784960,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 33311817728,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 33504493568,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.5.ffn_up.weight",
"offset": 33697169408,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 33889845248,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 33889878016,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.5.attn_output.weight",
"offset": 33896759296,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.5.attn_q.weight",
"offset": 33951809536,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.5.attn_v.weight",
"offset": 34006859776,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.50.attn_norm.weight",
"offset": 34013741056,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.50.ffn_down.weight",
"offset": 34013773824,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 34206449664,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.50.ffn_up.weight",
"offset": 34399125504,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 34591801344,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.50.attn_k.weight",
"offset": 34591834112,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.50.attn_output.weight",
"offset": 34598715392,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.50.attn_q.weight",
"offset": 34653765632,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.50.attn_v.weight",
"offset": 34708815872,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.51.attn_norm.weight",
"offset": 34715697152,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.51.ffn_down.weight",
"offset": 34715729920,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 34908405760,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.51.ffn_up.weight",
"offset": 35101081600,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 35293757440,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.51.attn_k.weight",
"offset": 35293790208,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.51.attn_output.weight",
"offset": 35300671488,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.51.attn_q.weight",
"offset": 35355721728,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.51.attn_v.weight",
"offset": 35410771968,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.52.attn_norm.weight",
"offset": 35417653248,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.52.ffn_down.weight",
"offset": 35417686016,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 35610361856,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.52.ffn_up.weight",
"offset": 35803037696,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 35995713536,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.52.attn_k.weight",
"offset": 35995746304,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.52.attn_output.weight",
"offset": 36002627584,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.52.attn_q.weight",
"offset": 36057677824,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.52.attn_v.weight",
"offset": 36112728064,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.53.attn_norm.weight",
"offset": 36119609344,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.53.ffn_down.weight",
"offset": 36119642112,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 36312317952,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.53.ffn_up.weight",
"offset": 36504993792,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 36697669632,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.53.attn_k.weight",
"offset": 36697702400,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.53.attn_output.weight",
"offset": 36704583680,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.53.attn_q.weight",
"offset": 36759633920,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.53.attn_v.weight",
"offset": 36814684160,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.54.attn_norm.weight",
"offset": 36821565440,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.54.ffn_down.weight",
"offset": 36821598208,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 37014274048,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.54.ffn_up.weight",
"offset": 37206949888,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 37399625728,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.54.attn_k.weight",
"offset": 37399658496,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.54.attn_output.weight",
"offset": 37406539776,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.54.attn_q.weight",
"offset": 37461590016,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.54.attn_v.weight",
"offset": 37516640256,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.55.attn_norm.weight",
"offset": 37523521536,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.55.ffn_down.weight",
"offset": 37523554304,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 37716230144,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.55.ffn_up.weight",
"offset": 37908905984,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 38101581824,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.55.attn_k.weight",
"offset": 38101614592,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.55.attn_output.weight",
"offset": 38108495872,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.55.attn_q.weight",
"offset": 38163546112,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.55.attn_v.weight",
"offset": 38218596352,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.56.attn_norm.weight",
"offset": 38225477632,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.56.ffn_down.weight",
"offset": 38225510400,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 38418186240,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.56.ffn_up.weight",
"offset": 38610862080,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 38803537920,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.56.attn_k.weight",
"offset": 38803570688,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.56.attn_output.weight",
"offset": 38810451968,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.56.attn_q.weight",
"offset": 38865502208,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.56.attn_v.weight",
"offset": 38920552448,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.57.attn_norm.weight",
"offset": 38927433728,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.57.ffn_down.weight",
"offset": 38927466496,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 39120142336,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.57.ffn_up.weight",
"offset": 39312818176,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 39505494016,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.57.attn_k.weight",
"offset": 39505526784,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.57.attn_output.weight",
"offset": 39512408064,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.57.attn_q.weight",
"offset": 39567458304,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.57.attn_v.weight",
"offset": 39622508544,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.58.attn_norm.weight",
"offset": 39629389824,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.58.ffn_down.weight",
"offset": 39629422592,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 39822098432,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.58.ffn_up.weight",
"offset": 40014774272,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 40207450112,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.58.attn_k.weight",
"offset": 40207482880,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.58.attn_output.weight",
"offset": 40214364160,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.58.attn_q.weight",
"offset": 40269414400,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.58.attn_v.weight",
"offset": 40324464640,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.59.attn_norm.weight",
"offset": 40331345920,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.59.ffn_down.weight",
"offset": 40331378688,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 40524054528,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.59.ffn_up.weight",
"offset": 40716730368,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 40909406208,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.59.attn_k.weight",
"offset": 40909438976,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.59.attn_output.weight",
"offset": 40916320256,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.59.attn_q.weight",
"offset": 40971370496,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.59.attn_v.weight",
"offset": 41026420736,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 41033302016,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 41033334784,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 41226010624,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.6.ffn_up.weight",
"offset": 41418686464,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 41611362304,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 41611395072,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.6.attn_output.weight",
"offset": 41618276352,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.6.attn_q.weight",
"offset": 41673326592,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.6.attn_v.weight",
"offset": 41728376832,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.60.attn_norm.weight",
"offset": 41735258112,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.60.ffn_down.weight",
"offset": 41735290880,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.60.ffn_gate.weight",
"offset": 41927966720,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.60.ffn_up.weight",
"offset": 42120642560,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.60.ffn_norm.weight",
"offset": 42313318400,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.60.attn_k.weight",
"offset": 42313351168,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.60.attn_output.weight",
"offset": 42320232448,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.60.attn_q.weight",
"offset": 42375282688,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.60.attn_v.weight",
"offset": 42430332928,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.61.attn_norm.weight",
"offset": 42437214208,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.61.ffn_down.weight",
"offset": 42437246976,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.61.ffn_gate.weight",
"offset": 42629922816,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.61.ffn_up.weight",
"offset": 42822598656,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.61.ffn_norm.weight",
"offset": 43015274496,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.61.attn_k.weight",
"offset": 43015307264,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.61.attn_output.weight",
"offset": 43022188544,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.61.attn_q.weight",
"offset": 43077238784,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.61.attn_v.weight",
"offset": 43132289024,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.62.attn_norm.weight",
"offset": 43139170304,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.62.ffn_down.weight",
"offset": 43139203072,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.62.ffn_gate.weight",
"offset": 43331878912,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.62.ffn_up.weight",
"offset": 43524554752,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.62.ffn_norm.weight",
"offset": 43717230592,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.62.attn_k.weight",
"offset": 43717263360,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.62.attn_output.weight",
"offset": 43724144640,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.62.attn_q.weight",
"offset": 43779194880,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.62.attn_v.weight",
"offset": 43834245120,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.63.attn_norm.weight",
"offset": 43841126400,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.63.ffn_down.weight",
"offset": 43841159168,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.63.ffn_gate.weight",
"offset": 44033835008,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.63.ffn_up.weight",
"offset": 44226510848,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.63.ffn_norm.weight",
"offset": 44419186688,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.63.attn_k.weight",
"offset": 44419219456,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.63.attn_output.weight",
"offset": 44426100736,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.63.attn_q.weight",
"offset": 44481150976,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.63.attn_v.weight",
"offset": 44536201216,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.64.attn_norm.weight",
"offset": 44543082496,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.64.ffn_down.weight",
"offset": 44543115264,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.64.ffn_gate.weight",
"offset": 44735791104,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.64.ffn_up.weight",
"offset": 44928466944,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.64.ffn_norm.weight",
"offset": 45121142784,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.64.attn_k.weight",
"offset": 45121175552,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.64.attn_output.weight",
"offset": 45128056832,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.64.attn_q.weight",
"offset": 45183107072,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.64.attn_v.weight",
"offset": 45238157312,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.65.attn_norm.weight",
"offset": 45245038592,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.65.ffn_down.weight",
"offset": 45245071360,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.65.ffn_gate.weight",
"offset": 45437747200,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.65.ffn_up.weight",
"offset": 45630423040,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.65.ffn_norm.weight",
"offset": 45823098880,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.65.attn_k.weight",
"offset": 45823131648,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.65.attn_output.weight",
"offset": 45830012928,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.65.attn_q.weight",
"offset": 45885063168,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.65.attn_v.weight",
"offset": 45940113408,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.66.attn_norm.weight",
"offset": 45946994688,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.66.ffn_down.weight",
"offset": 45947027456,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.66.ffn_gate.weight",
"offset": 46139703296,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.66.ffn_up.weight",
"offset": 46332379136,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.66.ffn_norm.weight",
"offset": 46525054976,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.66.attn_k.weight",
"offset": 46525087744,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.66.attn_output.weight",
"offset": 46531969024,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.66.attn_q.weight",
"offset": 46587019264,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.66.attn_v.weight",
"offset": 46642069504,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.67.attn_norm.weight",
"offset": 46648950784,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.67.ffn_down.weight",
"offset": 46648983552,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.67.ffn_gate.weight",
"offset": 46841659392,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.67.ffn_up.weight",
"offset": 47034335232,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.67.ffn_norm.weight",
"offset": 47227011072,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.67.attn_k.weight",
"offset": 47227043840,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.67.attn_output.weight",
"offset": 47233925120,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.67.attn_q.weight",
"offset": 47288975360,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.67.attn_v.weight",
"offset": 47344025600,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.68.attn_norm.weight",
"offset": 47350906880,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.68.ffn_down.weight",
"offset": 47350939648,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.68.ffn_gate.weight",
"offset": 47543615488,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.68.ffn_up.weight",
"offset": 47736291328,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.68.ffn_norm.weight",
"offset": 47928967168,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.68.attn_k.weight",
"offset": 47928999936,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.68.attn_output.weight",
"offset": 47935881216,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.68.attn_q.weight",
"offset": 47990931456,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.68.attn_v.weight",
"offset": 48045981696,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.69.attn_norm.weight",
"offset": 48052862976,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.69.ffn_down.weight",
"offset": 48052895744,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.69.ffn_gate.weight",
"offset": 48245571584,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.69.ffn_up.weight",
"offset": 48438247424,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.69.ffn_norm.weight",
"offset": 48630923264,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.69.attn_k.weight",
"offset": 48630956032,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.69.attn_output.weight",
"offset": 48637837312,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.69.attn_q.weight",
"offset": 48692887552,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.69.attn_v.weight",
"offset": 48747937792,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 48754819072,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 48754851840,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 48947527680,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.7.ffn_up.weight",
"offset": 49140203520,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 49332879360,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 49332912128,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.7.attn_output.weight",
"offset": 49339793408,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.7.attn_q.weight",
"offset": 49394843648,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.7.attn_v.weight",
"offset": 49449893888,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.70.attn_norm.weight",
"offset": 49456775168,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.70.ffn_down.weight",
"offset": 49456807936,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.70.ffn_gate.weight",
"offset": 49649483776,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.70.ffn_up.weight",
"offset": 49842159616,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.70.ffn_norm.weight",
"offset": 50034835456,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.70.attn_k.weight",
"offset": 50034868224,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.70.attn_output.weight",
"offset": 50041749504,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.70.attn_q.weight",
"offset": 50096799744,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.70.attn_v.weight",
"offset": 50151849984,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.71.attn_norm.weight",
"offset": 50158731264,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.71.ffn_down.weight",
"offset": 50158764032,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.71.ffn_gate.weight",
"offset": 50351439872,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.71.ffn_up.weight",
"offset": 50544115712,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.71.ffn_norm.weight",
"offset": 50736791552,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.71.attn_k.weight",
"offset": 50736824320,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.71.attn_output.weight",
"offset": 50743705600,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.71.attn_q.weight",
"offset": 50798755840,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.71.attn_v.weight",
"offset": 50853806080,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.72.attn_norm.weight",
"offset": 50860687360,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.72.ffn_down.weight",
"offset": 50860720128,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.72.ffn_gate.weight",
"offset": 51053395968,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.72.ffn_up.weight",
"offset": 51246071808,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.72.ffn_norm.weight",
"offset": 51438747648,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.72.attn_k.weight",
"offset": 51438780416,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.72.attn_output.weight",
"offset": 51445661696,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.72.attn_q.weight",
"offset": 51500711936,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.72.attn_v.weight",
"offset": 51555762176,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.73.attn_norm.weight",
"offset": 51562643456,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.73.ffn_down.weight",
"offset": 51562676224,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.73.ffn_gate.weight",
"offset": 51755352064,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.73.ffn_up.weight",
"offset": 51948027904,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.73.ffn_norm.weight",
"offset": 52140703744,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.73.attn_k.weight",
"offset": 52140736512,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.73.attn_output.weight",
"offset": 52147617792,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.73.attn_q.weight",
"offset": 52202668032,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.73.attn_v.weight",
"offset": 52257718272,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.74.attn_norm.weight",
"offset": 52264599552,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.74.ffn_down.weight",
"offset": 52264632320,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.74.ffn_gate.weight",
"offset": 52457308160,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.74.ffn_up.weight",
"offset": 52649984000,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.74.ffn_norm.weight",
"offset": 52842659840,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.74.attn_k.weight",
"offset": 52842692608,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.74.attn_output.weight",
"offset": 52849573888,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.74.attn_q.weight",
"offset": 52904624128,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.74.attn_v.weight",
"offset": 52959674368,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.75.attn_norm.weight",
"offset": 52966555648,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.75.ffn_down.weight",
"offset": 52966588416,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.75.ffn_gate.weight",
"offset": 53159264256,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.75.ffn_up.weight",
"offset": 53351940096,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.75.ffn_norm.weight",
"offset": 53544615936,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.75.attn_k.weight",
"offset": 53544648704,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.75.attn_output.weight",
"offset": 53551529984,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.75.attn_q.weight",
"offset": 53606580224,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.75.attn_v.weight",
"offset": 53661630464,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.76.attn_norm.weight",
"offset": 53668511744,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.76.ffn_down.weight",
"offset": 53668544512,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.76.ffn_gate.weight",
"offset": 53861220352,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.76.ffn_up.weight",
"offset": 54053896192,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.76.ffn_norm.weight",
"offset": 54246572032,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.76.attn_k.weight",
"offset": 54246604800,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.76.attn_output.weight",
"offset": 54253486080,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.76.attn_q.weight",
"offset": 54308536320,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.76.attn_v.weight",
"offset": 54363586560,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.77.attn_norm.weight",
"offset": 54370467840,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.77.ffn_down.weight",
"offset": 54370500608,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.77.ffn_gate.weight",
"offset": 54563176448,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.77.ffn_up.weight",
"offset": 54755852288,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.77.ffn_norm.weight",
"offset": 54948528128,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.77.attn_k.weight",
"offset": 54948560896,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.77.attn_output.weight",
"offset": 54955442176,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.77.attn_q.weight",
"offset": 55010492416,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.77.attn_v.weight",
"offset": 55065542656,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.78.attn_norm.weight",
"offset": 55072423936,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.78.ffn_down.weight",
"offset": 55072456704,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.78.ffn_gate.weight",
"offset": 55265132544,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.78.ffn_up.weight",
"offset": 55457808384,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.78.ffn_norm.weight",
"offset": 55650484224,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.78.attn_k.weight",
"offset": 55650516992,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.78.attn_output.weight",
"offset": 55657398272,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.78.attn_q.weight",
"offset": 55712448512,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.78.attn_v.weight",
"offset": 55767498752,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.79.attn_norm.weight",
"offset": 55774380032,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.79.ffn_down.weight",
"offset": 55774412800,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.79.ffn_gate.weight",
"offset": 55967088640,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.79.ffn_up.weight",
"offset": 56159764480,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.79.ffn_norm.weight",
"offset": 56352440320,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.79.attn_k.weight",
"offset": 56352473088,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.79.attn_output.weight",
"offset": 56359354368,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.79.attn_q.weight",
"offset": 56414404608,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.79.attn_v.weight",
"offset": 56469454848,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 56476336128,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 56476368896,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 56669044736,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.8.ffn_up.weight",
"offset": 56861720576,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 57054396416,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 57054429184,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.8.attn_output.weight",
"offset": 57061310464,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.8.attn_q.weight",
"offset": 57116360704,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.8.attn_v.weight",
"offset": 57171410944,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 57178292224,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 57178324992,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 57371000832,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.9.ffn_up.weight",
"offset": 57563676672,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 57756352512,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 57756385280,
"shape": [
8192,
1024
],
"size": 32768,
"type": 14
},
{
"name": "blk.9.attn_output.weight",
"offset": 57763266560,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.9.attn_q.weight",
"offset": 57818316800,
"shape": [
8192,
8192
],
"size": 55050240,
"type": 14
},
{
"name": "blk.9.attn_v.weight",
"offset": 57873367040,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "output_norm.weight",
"offset": 57880248320,
"shape": [
8192
],
"size": 6881280,
"type": 0
}
],
"version": 3
}