This model was created with the goal for a good llama 3 uncencored model with long context. At it worked like a charm.
70B
1,115 Pulls Updated 5 weeks ago
22d94db194a4 · 50GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 17,
"general.name": "Llama-3-70b-Uncensored-Lumi-Tess-gradient",
"general.quantization_version": 2,
"general.source.url": "https://huggingface.co/ryzen88/Llama-3-70b-Uncensored-Lumi-Tess-gradient",
"general.url": "https://huggingface.co/mradermacher/Llama-3-70b-Uncensored-Lumi-Tess-gradient-GGUF",
"llama.attention.head_count": 64,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 80,
"llama.context_length": 262144,
"llama.embedding_length": 8192,
"llama.feed_forward_length": 28672,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 207112200,
"llama.vocab_size": 128256,
"mradermacher.convert_type": "hf",
"mradermacher.quantize_version": "2",
"mradermacher.quantized_at": "2024-05-15T07:45:05+02:00",
"mradermacher.quantized_by": "mradermacher",
"mradermacher.quantized_on": "db1",
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.pre": "llama-bpe",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 70553706496,
"tensors": [
{
"name": "token_embd.weight",
"offset": 861880320,
"shape": [
8192,
128256
],
"size": 861880320,
"type": 13
},
{
"name": "blk.0.attn_norm.weight",
"offset": 1584218112,
"shape": [
8192
],
"size": 722337792,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 1584250880,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 1776926720,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.0.ffn_up.weight",
"offset": 1938407424,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 2099888128,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 2099920896,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.0.attn_output.weight",
"offset": 2105688064,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.0.attn_q.weight",
"offset": 2151825408,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.0.attn_v.weight",
"offset": 2197962752,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 2204844032,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 2204876800,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 2397552640,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.1.ffn_up.weight",
"offset": 2559033344,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 2720514048,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 2720546816,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.1.attn_output.weight",
"offset": 2726313984,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.1.attn_q.weight",
"offset": 2772451328,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.1.attn_v.weight",
"offset": 2818588672,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.10.attn_norm.weight",
"offset": 2825469952,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 2825502720,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 3018178560,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.10.ffn_up.weight",
"offset": 3179659264,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 3341139968,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 3341172736,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.10.attn_output.weight",
"offset": 3346939904,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.10.attn_q.weight",
"offset": 3393077248,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.10.attn_v.weight",
"offset": 3439214592,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 3446095872,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 3446128640,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 3638804480,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.11.ffn_up.weight",
"offset": 3800285184,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 3961765888,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 3961798656,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.11.attn_output.weight",
"offset": 3967565824,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.11.attn_q.weight",
"offset": 4013703168,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.11.attn_v.weight",
"offset": 4059840512,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 4066721792,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 4066754560,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 4259430400,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.12.ffn_up.weight",
"offset": 4420911104,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 4582391808,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 4582424576,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.12.attn_output.weight",
"offset": 4588191744,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.12.attn_q.weight",
"offset": 4634329088,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.12.attn_v.weight",
"offset": 4680466432,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.13.attn_norm.weight",
"offset": 4687347712,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 4687380480,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 4880056320,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.13.ffn_up.weight",
"offset": 5041537024,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 5203017728,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 5203050496,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.13.attn_output.weight",
"offset": 5208817664,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.13.attn_q.weight",
"offset": 5254955008,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.13.attn_v.weight",
"offset": 5301092352,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 5307973632,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 5308006400,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 5500682240,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.14.ffn_up.weight",
"offset": 5662162944,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 5823643648,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 5823676416,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.14.attn_output.weight",
"offset": 5829443584,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.14.attn_q.weight",
"offset": 5875580928,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.14.attn_v.weight",
"offset": 5921718272,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 5928599552,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 5928632320,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 6121308160,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.15.ffn_up.weight",
"offset": 6282788864,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 6444269568,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 6444302336,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.15.attn_output.weight",
"offset": 6450069504,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.15.attn_q.weight",
"offset": 6496206848,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.15.attn_v.weight",
"offset": 6542344192,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.16.attn_norm.weight",
"offset": 6549225472,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 6549258240,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 6741934080,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.16.ffn_up.weight",
"offset": 6903414784,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 7064895488,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 7064928256,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.16.attn_output.weight",
"offset": 7070695424,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.16.attn_q.weight",
"offset": 7116832768,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.16.attn_v.weight",
"offset": 7162970112,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 7169851392,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 7169884160,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 7362560000,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.17.ffn_up.weight",
"offset": 7524040704,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 7685521408,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 7685554176,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.17.attn_output.weight",
"offset": 7691321344,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.17.attn_q.weight",
"offset": 7737458688,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.17.attn_v.weight",
"offset": 7783596032,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 7790477312,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 7790510080,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 7951990784,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.18.ffn_up.weight",
"offset": 8113471488,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 8274952192,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 8274984960,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.18.attn_output.weight",
"offset": 8280752128,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.18.attn_q.weight",
"offset": 8326889472,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.18.attn_v.weight",
"offset": 8373026816,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.19.attn_norm.weight",
"offset": 8378793984,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 8378826752,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 8540307456,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.19.ffn_up.weight",
"offset": 8701788160,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 8863268864,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 8863301632,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.19.attn_output.weight",
"offset": 8869068800,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.19.attn_q.weight",
"offset": 8915206144,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.19.attn_v.weight",
"offset": 8961343488,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.2.attn_norm.weight",
"offset": 8967110656,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 8967143424,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 9159819264,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.2.ffn_up.weight",
"offset": 9321299968,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 9482780672,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 9482813440,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.2.attn_output.weight",
"offset": 9488580608,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.2.attn_q.weight",
"offset": 9534717952,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.2.attn_v.weight",
"offset": 9580855296,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 9587736576,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 9587769344,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 9749250048,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.20.ffn_up.weight",
"offset": 9910730752,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 10072211456,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 10072244224,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.20.attn_output.weight",
"offset": 10078011392,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.20.attn_q.weight",
"offset": 10124148736,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.20.attn_v.weight",
"offset": 10170286080,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.21.attn_norm.weight",
"offset": 10176053248,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 10176086016,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 10337566720,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.21.ffn_up.weight",
"offset": 10499047424,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 10660528128,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 10660560896,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.21.attn_output.weight",
"offset": 10666328064,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.21.attn_q.weight",
"offset": 10712465408,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.21.attn_v.weight",
"offset": 10758602752,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.22.attn_norm.weight",
"offset": 10764369920,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 10764402688,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 10957078528,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.22.ffn_up.weight",
"offset": 11118559232,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 11280039936,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 11280072704,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.22.attn_output.weight",
"offset": 11285839872,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.22.attn_q.weight",
"offset": 11331977216,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.22.attn_v.weight",
"offset": 11378114560,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 11384995840,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 11385028608,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 11546509312,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.23.ffn_up.weight",
"offset": 11707990016,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 11869470720,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 11869503488,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.23.attn_output.weight",
"offset": 11875270656,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.23.attn_q.weight",
"offset": 11921408000,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.23.attn_v.weight",
"offset": 11967545344,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.24.attn_norm.weight",
"offset": 11973312512,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 11973345280,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 12134825984,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.24.ffn_up.weight",
"offset": 12296306688,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 12457787392,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 12457820160,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.24.attn_output.weight",
"offset": 12463587328,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.24.attn_q.weight",
"offset": 12509724672,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.24.attn_v.weight",
"offset": 12555862016,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.25.attn_norm.weight",
"offset": 12561629184,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 12561661952,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 12754337792,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.25.ffn_up.weight",
"offset": 12915818496,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 13077299200,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 13077331968,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.25.attn_output.weight",
"offset": 13083099136,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.25.attn_q.weight",
"offset": 13129236480,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.25.attn_v.weight",
"offset": 13175373824,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.26.attn_norm.weight",
"offset": 13182255104,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 13182287872,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 13343768576,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.26.ffn_up.weight",
"offset": 13505249280,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 13666729984,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 13666762752,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.26.attn_output.weight",
"offset": 13672529920,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.26.attn_q.weight",
"offset": 13718667264,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.26.attn_v.weight",
"offset": 13764804608,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.27.attn_norm.weight",
"offset": 13770571776,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 13770604544,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 13932085248,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.27.ffn_up.weight",
"offset": 14093565952,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 14255046656,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 14255079424,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.27.attn_output.weight",
"offset": 14260846592,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.27.attn_q.weight",
"offset": 14306983936,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.27.attn_v.weight",
"offset": 14353121280,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.28.attn_norm.weight",
"offset": 14358888448,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 14358921216,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 14551597056,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.28.ffn_up.weight",
"offset": 14713077760,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 14874558464,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 14874591232,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.28.attn_output.weight",
"offset": 14880358400,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.28.attn_q.weight",
"offset": 14926495744,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.28.attn_v.weight",
"offset": 14972633088,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 14979514368,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 14979547136,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 15141027840,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.29.ffn_up.weight",
"offset": 15302508544,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 15463989248,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 15464022016,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.29.attn_output.weight",
"offset": 15469789184,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.29.attn_q.weight",
"offset": 15515926528,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.29.attn_v.weight",
"offset": 15562063872,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.3.attn_norm.weight",
"offset": 15567831040,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 15567863808,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 15729344512,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.3.ffn_up.weight",
"offset": 15890825216,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 16052305920,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 16052338688,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.3.attn_output.weight",
"offset": 16058105856,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.3.attn_q.weight",
"offset": 16104243200,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.3.attn_v.weight",
"offset": 16150380544,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.30.attn_norm.weight",
"offset": 16156147712,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 16156180480,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 16348856320,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.30.ffn_up.weight",
"offset": 16510337024,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 16671817728,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 16671850496,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.30.attn_output.weight",
"offset": 16677617664,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.30.attn_q.weight",
"offset": 16723755008,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.30.attn_v.weight",
"offset": 16769892352,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 16776773632,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 16776806400,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 16938287104,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.31.ffn_up.weight",
"offset": 17099767808,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 17261248512,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 17261281280,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.31.attn_output.weight",
"offset": 17267048448,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.31.attn_q.weight",
"offset": 17313185792,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.31.attn_v.weight",
"offset": 17359323136,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.32.attn_norm.weight",
"offset": 17365090304,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 17365123072,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 17526603776,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.32.ffn_up.weight",
"offset": 17688084480,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 17849565184,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 17849597952,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.32.attn_output.weight",
"offset": 17855365120,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.32.attn_q.weight",
"offset": 17901502464,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.32.attn_v.weight",
"offset": 17947639808,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.33.attn_norm.weight",
"offset": 17953406976,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 17953439744,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 18146115584,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.33.ffn_up.weight",
"offset": 18307596288,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 18469076992,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.33.attn_k.weight",
"offset": 18469109760,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.33.attn_output.weight",
"offset": 18474876928,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.33.attn_q.weight",
"offset": 18521014272,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.33.attn_v.weight",
"offset": 18567151616,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.34.attn_norm.weight",
"offset": 18574032896,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 18574065664,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 18735546368,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.34.ffn_up.weight",
"offset": 18897027072,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 19058507776,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 19058540544,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.34.attn_output.weight",
"offset": 19064307712,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.34.attn_q.weight",
"offset": 19110445056,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.34.attn_v.weight",
"offset": 19156582400,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.35.attn_norm.weight",
"offset": 19162349568,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 19162382336,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 19323863040,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.35.ffn_up.weight",
"offset": 19485343744,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 19646824448,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 19646857216,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.35.attn_output.weight",
"offset": 19652624384,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.35.attn_q.weight",
"offset": 19698761728,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.35.attn_v.weight",
"offset": 19744899072,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.36.attn_norm.weight",
"offset": 19750666240,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 19750699008,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 19943374848,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.36.ffn_up.weight",
"offset": 20104855552,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 20266336256,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 20266369024,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.36.attn_output.weight",
"offset": 20272136192,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.36.attn_q.weight",
"offset": 20318273536,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.36.attn_v.weight",
"offset": 20364410880,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.37.attn_norm.weight",
"offset": 20371292160,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 20371324928,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 20532805632,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.37.ffn_up.weight",
"offset": 20694286336,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 20855767040,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 20855799808,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.37.attn_output.weight",
"offset": 20861566976,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.37.attn_q.weight",
"offset": 20907704320,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.37.attn_v.weight",
"offset": 20953841664,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.38.attn_norm.weight",
"offset": 20959608832,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 20959641600,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 21121122304,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.38.ffn_up.weight",
"offset": 21282603008,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 21444083712,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.38.attn_k.weight",
"offset": 21444116480,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.38.attn_output.weight",
"offset": 21449883648,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.38.attn_q.weight",
"offset": 21496020992,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.38.attn_v.weight",
"offset": 21542158336,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.39.attn_norm.weight",
"offset": 21547925504,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 21547958272,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 21740634112,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.39.ffn_up.weight",
"offset": 21902114816,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 22063595520,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 22063628288,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.39.attn_output.weight",
"offset": 22069395456,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.39.attn_q.weight",
"offset": 22115532800,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.39.attn_v.weight",
"offset": 22161670144,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 22168551424,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 22168584192,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 22330064896,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.4.ffn_up.weight",
"offset": 22491545600,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 22653026304,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 22653059072,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.4.attn_output.weight",
"offset": 22658826240,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.4.attn_q.weight",
"offset": 22704963584,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.4.attn_v.weight",
"offset": 22751100928,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.40.attn_norm.weight",
"offset": 22756868096,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.40.ffn_down.weight",
"offset": 22756900864,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 22918381568,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.40.ffn_up.weight",
"offset": 23079862272,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 23241342976,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 23241375744,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.40.attn_output.weight",
"offset": 23247142912,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.40.attn_q.weight",
"offset": 23293280256,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.40.attn_v.weight",
"offset": 23339417600,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.41.attn_norm.weight",
"offset": 23345184768,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 23345217536,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 23537893376,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.41.ffn_up.weight",
"offset": 23699374080,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 23860854784,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.41.attn_k.weight",
"offset": 23860887552,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.41.attn_output.weight",
"offset": 23866654720,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.41.attn_q.weight",
"offset": 23912792064,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.41.attn_v.weight",
"offset": 23958929408,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.42.attn_norm.weight",
"offset": 23965810688,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 23965843456,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 24127324160,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.42.ffn_up.weight",
"offset": 24288804864,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 24450285568,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.42.attn_k.weight",
"offset": 24450318336,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.42.attn_output.weight",
"offset": 24456085504,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.42.attn_q.weight",
"offset": 24502222848,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.42.attn_v.weight",
"offset": 24548360192,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.43.attn_norm.weight",
"offset": 24554127360,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 24554160128,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 24715640832,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.43.ffn_up.weight",
"offset": 24877121536,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 25038602240,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 25038635008,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.43.attn_output.weight",
"offset": 25044402176,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.43.attn_q.weight",
"offset": 25090539520,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.43.attn_v.weight",
"offset": 25136676864,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.44.attn_norm.weight",
"offset": 25142444032,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 25142476800,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 25335152640,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.44.ffn_up.weight",
"offset": 25496633344,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 25658114048,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 25658146816,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.44.attn_output.weight",
"offset": 25663913984,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.44.attn_q.weight",
"offset": 25710051328,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.44.attn_v.weight",
"offset": 25756188672,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.45.attn_norm.weight",
"offset": 25763069952,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 25763102720,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 25924583424,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.45.ffn_up.weight",
"offset": 26086064128,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 26247544832,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 26247577600,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.45.attn_output.weight",
"offset": 26253344768,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.45.attn_q.weight",
"offset": 26299482112,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.45.attn_v.weight",
"offset": 26345619456,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.46.attn_norm.weight",
"offset": 26351386624,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 26351419392,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 26512900096,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.46.ffn_up.weight",
"offset": 26674380800,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 26835861504,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.46.attn_k.weight",
"offset": 26835894272,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.46.attn_output.weight",
"offset": 26841661440,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.46.attn_q.weight",
"offset": 26887798784,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.46.attn_v.weight",
"offset": 26933936128,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.47.attn_norm.weight",
"offset": 26939703296,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 26939736064,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 27132411904,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.47.ffn_up.weight",
"offset": 27293892608,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 27455373312,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.47.attn_k.weight",
"offset": 27455406080,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.47.attn_output.weight",
"offset": 27461173248,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.47.attn_q.weight",
"offset": 27507310592,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.47.attn_v.weight",
"offset": 27553447936,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.48.attn_norm.weight",
"offset": 27560329216,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.48.ffn_down.weight",
"offset": 27560361984,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 27721842688,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.48.ffn_up.weight",
"offset": 27883323392,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 28044804096,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.48.attn_k.weight",
"offset": 28044836864,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.48.attn_output.weight",
"offset": 28050604032,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.48.attn_q.weight",
"offset": 28096741376,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.48.attn_v.weight",
"offset": 28142878720,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.49.attn_norm.weight",
"offset": 28148645888,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.49.ffn_down.weight",
"offset": 28148678656,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 28310159360,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.49.ffn_up.weight",
"offset": 28471640064,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 28633120768,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.49.attn_k.weight",
"offset": 28633153536,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.49.attn_output.weight",
"offset": 28638920704,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.49.attn_q.weight",
"offset": 28685058048,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.49.attn_v.weight",
"offset": 28731195392,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.5.attn_norm.weight",
"offset": 28736962560,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 28736995328,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 28929671168,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.5.ffn_up.weight",
"offset": 29091151872,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 29252632576,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 29252665344,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.5.attn_output.weight",
"offset": 29258432512,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.5.attn_q.weight",
"offset": 29304569856,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.5.attn_v.weight",
"offset": 29350707200,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.50.attn_norm.weight",
"offset": 29357588480,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.50.ffn_down.weight",
"offset": 29357621248,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 29519101952,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.50.ffn_up.weight",
"offset": 29680582656,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 29842063360,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.50.attn_k.weight",
"offset": 29842096128,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.50.attn_output.weight",
"offset": 29847863296,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.50.attn_q.weight",
"offset": 29894000640,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.50.attn_v.weight",
"offset": 29940137984,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.51.attn_norm.weight",
"offset": 29945905152,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.51.ffn_down.weight",
"offset": 29945937920,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 30107418624,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.51.ffn_up.weight",
"offset": 30268899328,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 30430380032,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.51.attn_k.weight",
"offset": 30430412800,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.51.attn_output.weight",
"offset": 30436179968,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.51.attn_q.weight",
"offset": 30482317312,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.51.attn_v.weight",
"offset": 30528454656,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.52.attn_norm.weight",
"offset": 30534221824,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.52.ffn_down.weight",
"offset": 30534254592,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 30726930432,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.52.ffn_up.weight",
"offset": 30888411136,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 31049891840,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.52.attn_k.weight",
"offset": 31049924608,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.52.attn_output.weight",
"offset": 31055691776,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.52.attn_q.weight",
"offset": 31101829120,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.52.attn_v.weight",
"offset": 31147966464,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.53.attn_norm.weight",
"offset": 31154847744,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.53.ffn_down.weight",
"offset": 31154880512,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 31316361216,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.53.ffn_up.weight",
"offset": 31477841920,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 31639322624,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.53.attn_k.weight",
"offset": 31639355392,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.53.attn_output.weight",
"offset": 31645122560,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.53.attn_q.weight",
"offset": 31691259904,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.53.attn_v.weight",
"offset": 31737397248,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.54.attn_norm.weight",
"offset": 31743164416,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.54.ffn_down.weight",
"offset": 31743197184,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 31904677888,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.54.ffn_up.weight",
"offset": 32066158592,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 32227639296,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.54.attn_k.weight",
"offset": 32227672064,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.54.attn_output.weight",
"offset": 32233439232,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.54.attn_q.weight",
"offset": 32279576576,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.54.attn_v.weight",
"offset": 32325713920,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.55.attn_norm.weight",
"offset": 32331481088,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.55.ffn_down.weight",
"offset": 32331513856,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 32524189696,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.55.ffn_up.weight",
"offset": 32685670400,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 32847151104,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.55.attn_k.weight",
"offset": 32847183872,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.55.attn_output.weight",
"offset": 32852951040,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.55.attn_q.weight",
"offset": 32899088384,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.55.attn_v.weight",
"offset": 32945225728,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.56.attn_norm.weight",
"offset": 32952107008,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.56.ffn_down.weight",
"offset": 32952139776,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 33113620480,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.56.ffn_up.weight",
"offset": 33275101184,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 33436581888,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.56.attn_k.weight",
"offset": 33436614656,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.56.attn_output.weight",
"offset": 33442381824,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.56.attn_q.weight",
"offset": 33488519168,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.56.attn_v.weight",
"offset": 33534656512,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.57.attn_norm.weight",
"offset": 33540423680,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.57.ffn_down.weight",
"offset": 33540456448,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 33701937152,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.57.ffn_up.weight",
"offset": 33863417856,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 34024898560,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.57.attn_k.weight",
"offset": 34024931328,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.57.attn_output.weight",
"offset": 34030698496,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.57.attn_q.weight",
"offset": 34076835840,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.57.attn_v.weight",
"offset": 34122973184,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.58.attn_norm.weight",
"offset": 34128740352,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.58.ffn_down.weight",
"offset": 34128773120,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 34321448960,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.58.ffn_up.weight",
"offset": 34482929664,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 34644410368,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.58.attn_k.weight",
"offset": 34644443136,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.58.attn_output.weight",
"offset": 34650210304,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.58.attn_q.weight",
"offset": 34696347648,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.58.attn_v.weight",
"offset": 34742484992,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.59.attn_norm.weight",
"offset": 34749366272,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.59.ffn_down.weight",
"offset": 34749399040,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 34910879744,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.59.ffn_up.weight",
"offset": 35072360448,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 35233841152,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.59.attn_k.weight",
"offset": 35233873920,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.59.attn_output.weight",
"offset": 35239641088,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.59.attn_q.weight",
"offset": 35285778432,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.59.attn_v.weight",
"offset": 35331915776,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.6.attn_norm.weight",
"offset": 35337682944,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 35337715712,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 35499196416,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.6.ffn_up.weight",
"offset": 35660677120,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 35822157824,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 35822190592,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.6.attn_output.weight",
"offset": 35827957760,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.6.attn_q.weight",
"offset": 35874095104,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.6.attn_v.weight",
"offset": 35920232448,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.60.attn_norm.weight",
"offset": 35925999616,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.60.ffn_down.weight",
"offset": 35926032384,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.60.ffn_gate.weight",
"offset": 36118708224,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.60.ffn_up.weight",
"offset": 36280188928,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.60.ffn_norm.weight",
"offset": 36441669632,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.60.attn_k.weight",
"offset": 36441702400,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.60.attn_output.weight",
"offset": 36447469568,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.60.attn_q.weight",
"offset": 36493606912,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.60.attn_v.weight",
"offset": 36539744256,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.61.attn_norm.weight",
"offset": 36546625536,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.61.ffn_down.weight",
"offset": 36546658304,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.61.ffn_gate.weight",
"offset": 36708139008,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.61.ffn_up.weight",
"offset": 36869619712,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.61.ffn_norm.weight",
"offset": 37031100416,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.61.attn_k.weight",
"offset": 37031133184,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.61.attn_output.weight",
"offset": 37036900352,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.61.attn_q.weight",
"offset": 37083037696,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.61.attn_v.weight",
"offset": 37129175040,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.62.attn_norm.weight",
"offset": 37134942208,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.62.ffn_down.weight",
"offset": 37134974976,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.62.ffn_gate.weight",
"offset": 37296455680,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.62.ffn_up.weight",
"offset": 37457936384,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.62.ffn_norm.weight",
"offset": 37619417088,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.62.attn_k.weight",
"offset": 37619449856,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.62.attn_output.weight",
"offset": 37625217024,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.62.attn_q.weight",
"offset": 37671354368,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.62.attn_v.weight",
"offset": 37717491712,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.63.attn_norm.weight",
"offset": 37723258880,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.63.ffn_down.weight",
"offset": 37723291648,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.63.ffn_gate.weight",
"offset": 37915967488,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.63.ffn_up.weight",
"offset": 38077448192,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.63.ffn_norm.weight",
"offset": 38238928896,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.63.attn_k.weight",
"offset": 38238961664,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.63.attn_output.weight",
"offset": 38244728832,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.63.attn_q.weight",
"offset": 38290866176,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.63.attn_v.weight",
"offset": 38337003520,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.64.attn_norm.weight",
"offset": 38343884800,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.64.ffn_down.weight",
"offset": 38343917568,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.64.ffn_gate.weight",
"offset": 38505398272,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.64.ffn_up.weight",
"offset": 38666878976,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.64.ffn_norm.weight",
"offset": 38828359680,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.64.attn_k.weight",
"offset": 38828392448,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.64.attn_output.weight",
"offset": 38834159616,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.64.attn_q.weight",
"offset": 38880296960,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.64.attn_v.weight",
"offset": 38926434304,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.65.attn_norm.weight",
"offset": 38932201472,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.65.ffn_down.weight",
"offset": 38932234240,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.65.ffn_gate.weight",
"offset": 39093714944,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.65.ffn_up.weight",
"offset": 39255195648,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.65.ffn_norm.weight",
"offset": 39416676352,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.65.attn_k.weight",
"offset": 39416709120,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.65.attn_output.weight",
"offset": 39422476288,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.65.attn_q.weight",
"offset": 39468613632,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.65.attn_v.weight",
"offset": 39514750976,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.66.attn_norm.weight",
"offset": 39520518144,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.66.ffn_down.weight",
"offset": 39520550912,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.66.ffn_gate.weight",
"offset": 39713226752,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.66.ffn_up.weight",
"offset": 39874707456,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.66.ffn_norm.weight",
"offset": 40036188160,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.66.attn_k.weight",
"offset": 40036220928,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.66.attn_output.weight",
"offset": 40041988096,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.66.attn_q.weight",
"offset": 40088125440,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.66.attn_v.weight",
"offset": 40134262784,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.67.attn_norm.weight",
"offset": 40141144064,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.67.ffn_down.weight",
"offset": 40141176832,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.67.ffn_gate.weight",
"offset": 40302657536,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.67.ffn_up.weight",
"offset": 40464138240,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.67.ffn_norm.weight",
"offset": 40625618944,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.67.attn_k.weight",
"offset": 40625651712,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.67.attn_output.weight",
"offset": 40631418880,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.67.attn_q.weight",
"offset": 40677556224,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.67.attn_v.weight",
"offset": 40723693568,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.68.attn_norm.weight",
"offset": 40729460736,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.68.ffn_down.weight",
"offset": 40729493504,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.68.ffn_gate.weight",
"offset": 40890974208,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.68.ffn_up.weight",
"offset": 41052454912,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.68.ffn_norm.weight",
"offset": 41213935616,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.68.attn_k.weight",
"offset": 41213968384,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.68.attn_output.weight",
"offset": 41219735552,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.68.attn_q.weight",
"offset": 41265872896,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.68.attn_v.weight",
"offset": 41312010240,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.69.attn_norm.weight",
"offset": 41317777408,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.69.ffn_down.weight",
"offset": 41317810176,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.69.ffn_gate.weight",
"offset": 41510486016,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.69.ffn_up.weight",
"offset": 41671966720,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.69.ffn_norm.weight",
"offset": 41833447424,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.69.attn_k.weight",
"offset": 41833480192,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.69.attn_output.weight",
"offset": 41839247360,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.69.attn_q.weight",
"offset": 41885384704,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.69.attn_v.weight",
"offset": 41931522048,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 41938403328,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 41938436096,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 42099916800,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.7.ffn_up.weight",
"offset": 42261397504,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 42422878208,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 42422910976,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.7.attn_output.weight",
"offset": 42428678144,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.7.attn_q.weight",
"offset": 42474815488,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.7.attn_v.weight",
"offset": 42520952832,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.70.attn_norm.weight",
"offset": 42526720000,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.70.ffn_down.weight",
"offset": 42526752768,
"shape": [
28672,
8192
],
"size": 32768,
"type": 13
},
{
"name": "blk.70.ffn_gate.weight",
"offset": 42688233472,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.70.ffn_up.weight",
"offset": 42849714176,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.70.ffn_norm.weight",
"offset": 43011194880,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.70.attn_k.weight",
"offset": 43011227648,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.70.attn_output.weight",
"offset": 43016994816,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.70.attn_q.weight",
"offset": 43063132160,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.70.attn_v.weight",
"offset": 43109269504,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 13
},
{
"name": "blk.71.attn_norm.weight",
"offset": 43115036672,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.71.ffn_down.weight",
"offset": 43115069440,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.71.ffn_gate.weight",
"offset": 43307745280,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.71.ffn_up.weight",
"offset": 43469225984,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.71.ffn_norm.weight",
"offset": 43630706688,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.71.attn_k.weight",
"offset": 43630739456,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.71.attn_output.weight",
"offset": 43636506624,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.71.attn_q.weight",
"offset": 43682643968,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.71.attn_v.weight",
"offset": 43728781312,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.72.attn_norm.weight",
"offset": 43735662592,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.72.ffn_down.weight",
"offset": 43735695360,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.72.ffn_gate.weight",
"offset": 43928371200,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.72.ffn_up.weight",
"offset": 44089851904,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.72.ffn_norm.weight",
"offset": 44251332608,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.72.attn_k.weight",
"offset": 44251365376,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.72.attn_output.weight",
"offset": 44257132544,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.72.attn_q.weight",
"offset": 44303269888,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.72.attn_v.weight",
"offset": 44349407232,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.73.attn_norm.weight",
"offset": 44356288512,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.73.ffn_down.weight",
"offset": 44356321280,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.73.ffn_gate.weight",
"offset": 44548997120,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.73.ffn_up.weight",
"offset": 44710477824,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.73.ffn_norm.weight",
"offset": 44871958528,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.73.attn_k.weight",
"offset": 44871991296,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.73.attn_output.weight",
"offset": 44877758464,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.73.attn_q.weight",
"offset": 44923895808,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.73.attn_v.weight",
"offset": 44970033152,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.74.attn_norm.weight",
"offset": 44976914432,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.74.ffn_down.weight",
"offset": 44976947200,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.74.ffn_gate.weight",
"offset": 45169623040,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.74.ffn_up.weight",
"offset": 45331103744,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.74.ffn_norm.weight",
"offset": 45492584448,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.74.attn_k.weight",
"offset": 45492617216,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.74.attn_output.weight",
"offset": 45498384384,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.74.attn_q.weight",
"offset": 45544521728,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.74.attn_v.weight",
"offset": 45590659072,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.75.attn_norm.weight",
"offset": 45597540352,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.75.ffn_down.weight",
"offset": 45597573120,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.75.ffn_gate.weight",
"offset": 45790248960,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.75.ffn_up.weight",
"offset": 45951729664,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.75.ffn_norm.weight",
"offset": 46113210368,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.75.attn_k.weight",
"offset": 46113243136,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.75.attn_output.weight",
"offset": 46119010304,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.75.attn_q.weight",
"offset": 46165147648,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.75.attn_v.weight",
"offset": 46211284992,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.76.attn_norm.weight",
"offset": 46218166272,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.76.ffn_down.weight",
"offset": 46218199040,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.76.ffn_gate.weight",
"offset": 46410874880,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.76.ffn_up.weight",
"offset": 46572355584,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.76.ffn_norm.weight",
"offset": 46733836288,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.76.attn_k.weight",
"offset": 46733869056,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.76.attn_output.weight",
"offset": 46739636224,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.76.attn_q.weight",
"offset": 46785773568,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.76.attn_v.weight",
"offset": 46831910912,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.77.attn_norm.weight",
"offset": 46838792192,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.77.ffn_down.weight",
"offset": 46838824960,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.77.ffn_gate.weight",
"offset": 47031500800,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.77.ffn_up.weight",
"offset": 47192981504,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.77.ffn_norm.weight",
"offset": 47354462208,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.77.attn_k.weight",
"offset": 47354494976,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.77.attn_output.weight",
"offset": 47360262144,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.77.attn_q.weight",
"offset": 47406399488,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.77.attn_v.weight",
"offset": 47452536832,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.78.attn_norm.weight",
"offset": 47459418112,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.78.ffn_down.weight",
"offset": 47459450880,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.78.ffn_gate.weight",
"offset": 47652126720,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.78.ffn_up.weight",
"offset": 47813607424,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.78.ffn_norm.weight",
"offset": 47975088128,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.78.attn_k.weight",
"offset": 47975120896,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.78.attn_output.weight",
"offset": 47980888064,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.78.attn_q.weight",
"offset": 48027025408,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.78.attn_v.weight",
"offset": 48073162752,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.79.attn_norm.weight",
"offset": 48080044032,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.79.ffn_down.weight",
"offset": 48080076800,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.79.ffn_gate.weight",
"offset": 48272752640,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.79.ffn_up.weight",
"offset": 48434233344,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.79.ffn_norm.weight",
"offset": 48595714048,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.79.attn_k.weight",
"offset": 48595746816,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.79.attn_output.weight",
"offset": 48601513984,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.79.attn_q.weight",
"offset": 48647651328,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.79.attn_v.weight",
"offset": 48693788672,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 48700669952,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 48700702720,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 48893378560,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.8.ffn_up.weight",
"offset": 49054859264,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 49216339968,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 49216372736,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.8.attn_output.weight",
"offset": 49222139904,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.8.attn_q.weight",
"offset": 49268277248,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.8.attn_v.weight",
"offset": 49314414592,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 49321295872,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 49321328640,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 49514004480,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 13
},
{
"name": "blk.9.ffn_up.weight",
"offset": 49675485184,
"shape": [
8192,
28672
],
"size": 161480704,
"type": 13
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 49836965888,
"shape": [
8192
],
"size": 161480704,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 49836998656,
"shape": [
8192,
1024
],
"size": 32768,
"type": 13
},
{
"name": "blk.9.attn_output.weight",
"offset": 49842765824,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 13
},
{
"name": "blk.9.attn_q.weight",
"offset": 49888903168,
"shape": [
8192,
8192
],
"size": 46137344,
"type": 13
},
{
"name": "blk.9.attn_v.weight",
"offset": 49935040512,
"shape": [
8192,
1024
],
"size": 46137344,
"type": 14
},
{
"name": "output_norm.weight",
"offset": 49941921792,
"shape": [
8192
],
"size": 6881280,
"type": 0
}
],
"version": 3
}