latest
26GB
Qra is foundation language model trained with causal language modeling objective on a large corpus of texts.
13B
35 Pulls Updated 2 months ago
d38158526827 · 26GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 1,
"general.name": "LLaMA v2",
"llama.attention.head_count": 40,
"llama.attention.head_count_kv": 40,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 40,
"llama.context_length": 4096,
"llama.embedding_length": 5120,
"llama.feed_forward_length": 13824,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 10000,
"tokenizer.ggml.add_bos_token": true,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 2,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 2,
"tokenizer.ggml.scores": "... (32000 values)",
"tokenizer.ggml.token_type": "... (32000 values)",
"tokenizer.ggml.tokens": "... (32000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 13015864320,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 327680000,
"shape": [
5120
],
"size": 327680000,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 327700480,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 469258240,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.0.ffn_up.weight",
"offset": 610816000,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 752373760,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 752394240,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.0.attn_output.weight",
"offset": 804823040,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.0.attn_q.weight",
"offset": 857251840,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.0.attn_v.weight",
"offset": 909680640,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.1.attn_norm.weight",
"offset": 962109440,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 962129920,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 1103687680,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.1.ffn_up.weight",
"offset": 1245245440,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 1386803200,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 1386823680,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.1.attn_output.weight",
"offset": 1439252480,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.1.attn_q.weight",
"offset": 1491681280,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.1.attn_v.weight",
"offset": 1544110080,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.2.attn_norm.weight",
"offset": 1596538880,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 1596559360,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 1738117120,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.2.ffn_up.weight",
"offset": 1879674880,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 2021232640,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 2021253120,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.2.attn_output.weight",
"offset": 2073681920,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.2.attn_q.weight",
"offset": 2126110720,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.2.attn_v.weight",
"offset": 2178539520,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.3.attn_norm.weight",
"offset": 2230968320,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 2230988800,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 2372546560,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.3.ffn_up.weight",
"offset": 2514104320,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 2655662080,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 2655682560,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.3.attn_output.weight",
"offset": 2708111360,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.3.attn_q.weight",
"offset": 2760540160,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.3.attn_v.weight",
"offset": 2812968960,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.4.attn_norm.weight",
"offset": 2865397760,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 2865418240,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 3006976000,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.4.ffn_up.weight",
"offset": 3148533760,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 3290091520,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 3290112000,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.4.attn_output.weight",
"offset": 3342540800,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.4.attn_q.weight",
"offset": 3394969600,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.4.attn_v.weight",
"offset": 3447398400,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.5.attn_norm.weight",
"offset": 3499827200,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 3499847680,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 3641405440,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.5.ffn_up.weight",
"offset": 3782963200,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 3924520960,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 3924541440,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.5.attn_output.weight",
"offset": 3976970240,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.5.attn_q.weight",
"offset": 4029399040,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.5.attn_v.weight",
"offset": 4081827840,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.6.attn_norm.weight",
"offset": 4134256640,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 4134277120,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 4275834880,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.6.ffn_up.weight",
"offset": 4417392640,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 4558950400,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 4558970880,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.6.attn_output.weight",
"offset": 4611399680,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.6.attn_q.weight",
"offset": 4663828480,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.6.attn_v.weight",
"offset": 4716257280,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.7.attn_k.weight",
"offset": 4768686080,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.7.attn_output.weight",
"offset": 4821114880,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.7.attn_q.weight",
"offset": 4873543680,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.7.attn_v.weight",
"offset": 4925972480,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.10.attn_norm.weight",
"offset": 4978401280,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 4978421760,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 5119979520,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.10.ffn_up.weight",
"offset": 5261537280,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 5403095040,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 5403115520,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.10.attn_output.weight",
"offset": 5455544320,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.10.attn_q.weight",
"offset": 5507973120,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.10.attn_v.weight",
"offset": 5560401920,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.11.attn_norm.weight",
"offset": 5612830720,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 5612851200,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 5754408960,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.11.ffn_up.weight",
"offset": 5895966720,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 6037524480,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 6037544960,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.11.attn_output.weight",
"offset": 6089973760,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.11.attn_q.weight",
"offset": 6142402560,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.11.attn_v.weight",
"offset": 6194831360,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.12.attn_norm.weight",
"offset": 6247260160,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 6247280640,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 6388838400,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.12.ffn_up.weight",
"offset": 6530396160,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 6671953920,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 6671974400,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.12.attn_output.weight",
"offset": 6724403200,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.12.attn_q.weight",
"offset": 6776832000,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.12.attn_v.weight",
"offset": 6829260800,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.13.attn_norm.weight",
"offset": 6881689600,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 6881710080,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 7023267840,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.13.ffn_up.weight",
"offset": 7164825600,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 7306383360,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 7306403840,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.13.attn_output.weight",
"offset": 7358832640,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.13.attn_q.weight",
"offset": 7411261440,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.13.attn_v.weight",
"offset": 7463690240,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.14.attn_norm.weight",
"offset": 7516119040,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 7516139520,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 7657697280,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.14.ffn_up.weight",
"offset": 7799255040,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 7940812800,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 7940833280,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.14.attn_output.weight",
"offset": 7993262080,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.14.attn_q.weight",
"offset": 8045690880,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.14.attn_v.weight",
"offset": 8098119680,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.15.attn_k.weight",
"offset": 8150548480,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.15.attn_q.weight",
"offset": 8202977280,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.7.attn_norm.weight",
"offset": 8255406080,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 8255426560,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 8396984320,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.7.ffn_up.weight",
"offset": 8538542080,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 8680099840,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.8.attn_norm.weight",
"offset": 8680120320,
"shape": [
5120
],
"size": 20480,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 8680140800,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 8821698560,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.8.ffn_up.weight",
"offset": 8963256320,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 9104814080,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 9104834560,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.8.attn_output.weight",
"offset": 9157263360,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.8.attn_q.weight",
"offset": 9209692160,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.8.attn_v.weight",
"offset": 9262120960,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.9.attn_norm.weight",
"offset": 9314549760,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 9314570240,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 9456128000,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.9.ffn_up.weight",
"offset": 9597685760,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 9739243520,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 9739264000,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.9.attn_output.weight",
"offset": 9791692800,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.9.attn_q.weight",
"offset": 9844121600,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.9.attn_v.weight",
"offset": 9896550400,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.15.attn_norm.weight",
"offset": 9948979200,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 9948999680,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 10090557440,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.15.ffn_up.weight",
"offset": 10232115200,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 10373672960,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.15.attn_output.weight",
"offset": 10373693440,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.15.attn_v.weight",
"offset": 10426122240,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.16.attn_norm.weight",
"offset": 10478551040,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 10478571520,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 10620129280,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.16.ffn_up.weight",
"offset": 10761687040,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 10903244800,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 10903265280,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.16.attn_output.weight",
"offset": 10955694080,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.16.attn_q.weight",
"offset": 11008122880,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.16.attn_v.weight",
"offset": 11060551680,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.17.attn_norm.weight",
"offset": 11112980480,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 11113000960,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 11254558720,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.17.ffn_up.weight",
"offset": 11396116480,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 11537674240,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 11537694720,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.17.attn_output.weight",
"offset": 11590123520,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.17.attn_q.weight",
"offset": 11642552320,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.17.attn_v.weight",
"offset": 11694981120,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.18.attn_norm.weight",
"offset": 11747409920,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 11747430400,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 11888988160,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.18.ffn_up.weight",
"offset": 12030545920,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 12172103680,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 12172124160,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.18.attn_output.weight",
"offset": 12224552960,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.18.attn_q.weight",
"offset": 12276981760,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.18.attn_v.weight",
"offset": 12329410560,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.19.attn_norm.weight",
"offset": 12381839360,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 12381859840,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 12523417600,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.19.ffn_up.weight",
"offset": 12664975360,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 12806533120,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 12806553600,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.19.attn_output.weight",
"offset": 12858982400,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.19.attn_q.weight",
"offset": 12911411200,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.19.attn_v.weight",
"offset": 12963840000,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.20.attn_norm.weight",
"offset": 13016268800,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 13016289280,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 13157847040,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.20.ffn_up.weight",
"offset": 13299404800,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 13440962560,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 13440983040,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.20.attn_output.weight",
"offset": 13493411840,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.20.attn_q.weight",
"offset": 13545840640,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.20.attn_v.weight",
"offset": 13598269440,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.21.attn_norm.weight",
"offset": 13650698240,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 13650718720,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 13792276480,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.21.ffn_up.weight",
"offset": 13933834240,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 14075392000,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 14075412480,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.21.attn_output.weight",
"offset": 14127841280,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.21.attn_q.weight",
"offset": 14180270080,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.21.attn_v.weight",
"offset": 14232698880,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.22.attn_norm.weight",
"offset": 14285127680,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 14285148160,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 14426705920,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.22.ffn_up.weight",
"offset": 14568263680,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 14709821440,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 14709841920,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.22.attn_output.weight",
"offset": 14762270720,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.22.attn_q.weight",
"offset": 14814699520,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.22.attn_v.weight",
"offset": 14867128320,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.23.attn_norm.weight",
"offset": 14919557120,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 14919577600,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 15061135360,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.23.ffn_up.weight",
"offset": 15202693120,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 15344250880,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 15344271360,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.23.attn_output.weight",
"offset": 15396700160,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.23.attn_q.weight",
"offset": 15449128960,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.23.attn_v.weight",
"offset": 15501557760,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.24.attn_norm.weight",
"offset": 15553986560,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 15554007040,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 15695564800,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.24.ffn_up.weight",
"offset": 15837122560,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 15978680320,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 15978700800,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.24.attn_output.weight",
"offset": 16031129600,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.24.attn_q.weight",
"offset": 16083558400,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.24.attn_v.weight",
"offset": 16135987200,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.25.attn_norm.weight",
"offset": 16188416000,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 16188436480,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 16329994240,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.25.ffn_up.weight",
"offset": 16471552000,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 16613109760,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 16613130240,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.25.attn_output.weight",
"offset": 16665559040,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.25.attn_q.weight",
"offset": 16717987840,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.25.attn_v.weight",
"offset": 16770416640,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.26.attn_norm.weight",
"offset": 16822845440,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 16822865920,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 16964423680,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.26.ffn_up.weight",
"offset": 17105981440,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 17247539200,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 17247559680,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.26.attn_output.weight",
"offset": 17299988480,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.26.attn_q.weight",
"offset": 17352417280,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.26.attn_v.weight",
"offset": 17404846080,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.27.attn_norm.weight",
"offset": 17457274880,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 17457295360,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 17598853120,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.27.ffn_up.weight",
"offset": 17740410880,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 17881968640,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 17881989120,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.27.attn_output.weight",
"offset": 17934417920,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.27.attn_q.weight",
"offset": 17986846720,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.27.attn_v.weight",
"offset": 18039275520,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.28.attn_norm.weight",
"offset": 18091704320,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 18091724800,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 18233282560,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.28.ffn_up.weight",
"offset": 18374840320,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 18516398080,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 18516418560,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.28.attn_output.weight",
"offset": 18568847360,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.28.attn_q.weight",
"offset": 18621276160,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.28.attn_v.weight",
"offset": 18673704960,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.29.attn_norm.weight",
"offset": 18726133760,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 18726154240,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 18867712000,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.29.ffn_up.weight",
"offset": 19009269760,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 19150827520,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 19150848000,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.29.attn_output.weight",
"offset": 19203276800,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.29.attn_q.weight",
"offset": 19255705600,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.29.attn_v.weight",
"offset": 19308134400,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 19360563200,
"shape": [
5120,
13824
],
"size": 52428800,
"type": 1
},
{
"name": "blk.30.ffn_up.weight",
"offset": 19502120960,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.30.attn_k.weight",
"offset": 19643678720,
"shape": [
5120,
5120
],
"size": 141557760,
"type": 1
},
{
"name": "blk.30.attn_output.weight",
"offset": 19696107520,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.30.attn_q.weight",
"offset": 19748536320,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.30.attn_v.weight",
"offset": 19800965120,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.30.attn_norm.weight",
"offset": 19853393920,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 19853414400,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 19994972160,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.31.attn_norm.weight",
"offset": 19994992640,
"shape": [
5120
],
"size": 20480,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 19995013120,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 20136570880,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.31.ffn_up.weight",
"offset": 20278128640,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 20419686400,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 20419706880,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.31.attn_output.weight",
"offset": 20472135680,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.31.attn_q.weight",
"offset": 20524564480,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.31.attn_v.weight",
"offset": 20576993280,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.32.attn_norm.weight",
"offset": 20629422080,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 20629442560,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 20771000320,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.32.ffn_up.weight",
"offset": 20912558080,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 21054115840,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 21054136320,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.32.attn_output.weight",
"offset": 21106565120,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.32.attn_q.weight",
"offset": 21158993920,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.32.attn_v.weight",
"offset": 21211422720,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.33.attn_norm.weight",
"offset": 21263851520,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 21263872000,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 21405429760,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.33.ffn_up.weight",
"offset": 21546987520,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 21688545280,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.33.attn_k.weight",
"offset": 21688565760,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.33.attn_output.weight",
"offset": 21740994560,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.33.attn_q.weight",
"offset": 21793423360,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.33.attn_v.weight",
"offset": 21845852160,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.34.attn_norm.weight",
"offset": 21898280960,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 21898301440,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 22039859200,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.34.ffn_up.weight",
"offset": 22181416960,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 22322974720,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 22322995200,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.34.attn_output.weight",
"offset": 22375424000,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.34.attn_q.weight",
"offset": 22427852800,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.34.attn_v.weight",
"offset": 22480281600,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.35.attn_norm.weight",
"offset": 22532710400,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 22532730880,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 22674288640,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.35.ffn_up.weight",
"offset": 22815846400,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 22957404160,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 22957424640,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.35.attn_output.weight",
"offset": 23009853440,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.35.attn_q.weight",
"offset": 23062282240,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.35.attn_v.weight",
"offset": 23114711040,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.36.attn_norm.weight",
"offset": 23167139840,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 23167160320,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 23308718080,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.36.ffn_up.weight",
"offset": 23450275840,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 23591833600,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 23591854080,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.36.attn_output.weight",
"offset": 23644282880,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.36.attn_q.weight",
"offset": 23696711680,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.36.attn_v.weight",
"offset": 23749140480,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.37.attn_norm.weight",
"offset": 23801569280,
"shape": [
5120
],
"size": 52428800,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 23801589760,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 23943147520,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.37.ffn_up.weight",
"offset": 24084705280,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 24226263040,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 24226283520,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.37.attn_output.weight",
"offset": 24278712320,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.37.attn_q.weight",
"offset": 24331141120,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.37.attn_v.weight",
"offset": 24383569920,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 24435998720,
"shape": [
5120,
13824
],
"size": 52428800,
"type": 1
},
{
"name": "blk.38.attn_k.weight",
"offset": 24577556480,
"shape": [
5120,
5120
],
"size": 141557760,
"type": 1
},
{
"name": "blk.38.attn_output.weight",
"offset": 24629985280,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.38.attn_q.weight",
"offset": 24682414080,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.38.attn_v.weight",
"offset": 24734842880,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "output.weight",
"offset": 24787271680,
"shape": [
5120,
32000
],
"size": 52428800,
"type": 1
},
{
"name": "blk.38.attn_norm.weight",
"offset": 25114951680,
"shape": [
5120
],
"size": 327680000,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 25114972160,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.38.ffn_up.weight",
"offset": 25256529920,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 25398087680,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.39.attn_norm.weight",
"offset": 25398108160,
"shape": [
5120
],
"size": 20480,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 25398128640,
"shape": [
13824,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 25539686400,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.39.ffn_up.weight",
"offset": 25681244160,
"shape": [
5120,
13824
],
"size": 141557760,
"type": 1
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 25822801920,
"shape": [
5120
],
"size": 141557760,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 25822822400,
"shape": [
5120,
5120
],
"size": 20480,
"type": 1
},
{
"name": "blk.39.attn_output.weight",
"offset": 25875251200,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.39.attn_q.weight",
"offset": 25927680000,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "blk.39.attn_v.weight",
"offset": 25980108800,
"shape": [
5120,
5120
],
"size": 52428800,
"type": 1
},
{
"name": "output_norm.weight",
"offset": 26032537600,
"shape": [
5120
],
"size": 52428800,
"type": 0
}
],
"version": 3
}