latest
26GB
Qra is foundation language model trained with causal language modeling objective on a large corpus of texts.
13B
36 Pulls Updated 2 months ago
6e6374f25352 · 7.4GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 2,
"general.name": "LLaMA v2",
"general.quantization_version": 2,
"llama.attention.head_count": 40,
"llama.attention.head_count_kv": 40,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 40,
"llama.context_length": 4096,
"llama.embedding_length": 5120,
"llama.feed_forward_length": 13824,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 10000,
"tokenizer.ggml.add_bos_token": true,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 2,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 2,
"tokenizer.ggml.scores": "... (32000 values)",
"tokenizer.ggml.token_type": "... (32000 values)",
"tokenizer.ggml.tokens": "... (32000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 13015864320,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 92160000,
"shape": [
5120
],
"size": 92160000,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 92180480,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 131993600,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.0.ffn_up.weight",
"offset": 171806720,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 211619840,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 211640320,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.0.attn_output.weight",
"offset": 226385920,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.0.attn_q.weight",
"offset": 241131520,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.0.attn_v.weight",
"offset": 255877120,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.1.attn_norm.weight",
"offset": 270622720,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 270643200,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 310456320,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.1.ffn_up.weight",
"offset": 350269440,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 390082560,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 390103040,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.1.attn_output.weight",
"offset": 404848640,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.1.attn_q.weight",
"offset": 419594240,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.1.attn_v.weight",
"offset": 434339840,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.2.attn_norm.weight",
"offset": 449085440,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 449105920,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 488919040,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.2.ffn_up.weight",
"offset": 528732160,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 568545280,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 568565760,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.2.attn_output.weight",
"offset": 583311360,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.2.attn_q.weight",
"offset": 598056960,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.2.attn_v.weight",
"offset": 612802560,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.3.attn_norm.weight",
"offset": 627548160,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 627568640,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 667381760,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.3.ffn_up.weight",
"offset": 707194880,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 747008000,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 747028480,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.3.attn_output.weight",
"offset": 761774080,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.3.attn_q.weight",
"offset": 776519680,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.3.attn_v.weight",
"offset": 791265280,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.4.attn_norm.weight",
"offset": 806010880,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 806031360,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 845844480,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.4.ffn_up.weight",
"offset": 885657600,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 925470720,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 925491200,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.4.attn_output.weight",
"offset": 940236800,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.4.attn_q.weight",
"offset": 954982400,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.4.attn_v.weight",
"offset": 969728000,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.5.attn_norm.weight",
"offset": 984473600,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 984494080,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1024307200,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1064120320,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1103933440,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1103953920,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.5.attn_output.weight",
"offset": 1118699520,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.5.attn_q.weight",
"offset": 1133445120,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.5.attn_v.weight",
"offset": 1148190720,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1162936320,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1162956800,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 1202769920,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.6.ffn_up.weight",
"offset": 1242583040,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 1282396160,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 1282416640,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.6.attn_output.weight",
"offset": 1297162240,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.6.attn_q.weight",
"offset": 1311907840,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.6.attn_v.weight",
"offset": 1326653440,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.7.attn_k.weight",
"offset": 1341399040,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.7.attn_output.weight",
"offset": 1356144640,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.7.attn_q.weight",
"offset": 1370890240,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.7.attn_v.weight",
"offset": 1385635840,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.10.attn_norm.weight",
"offset": 1400381440,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 1400401920,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 1440215040,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.10.ffn_up.weight",
"offset": 1480028160,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 1519841280,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 1519861760,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.10.attn_output.weight",
"offset": 1534607360,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.10.attn_q.weight",
"offset": 1549352960,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.10.attn_v.weight",
"offset": 1564098560,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.11.attn_norm.weight",
"offset": 1578844160,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 1578864640,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 1618677760,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.11.ffn_up.weight",
"offset": 1658490880,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 1698304000,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 1698324480,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.11.attn_output.weight",
"offset": 1713070080,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.11.attn_q.weight",
"offset": 1727815680,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.11.attn_v.weight",
"offset": 1742561280,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.12.attn_norm.weight",
"offset": 1757306880,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "\u0000\u0000\u0014\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0014\u0000\u0000\u0000\u0000\u0000\u0000\u0002\u0000\u0000\u0000\u0000",
"offset": 1757327360,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 1797140480,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.12.ffn_up.weight",
"offset": 1836953600,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 1876766720,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 1876787200,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.12.attn_output.weight",
"offset": 1891532800,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.12.attn_q.weight",
"offset": 1906278400,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.12.attn_v.weight",
"offset": 1921024000,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.13.attn_norm.weight",
"offset": 1935769600,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 1935790080,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 1975603200,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.13.ffn_up.weight",
"offset": 2015416320,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 2055229440,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 2055249920,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.13.attn_output.weight",
"offset": 2069995520,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.13.attn_q.weight",
"offset": 2084741120,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.13.attn_v.weight",
"offset": 2099486720,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.14.attn_norm.weight",
"offset": 2114232320,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 2114252800,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 2154065920,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.14.ffn_up.weight",
"offset": 2193879040,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 2233692160,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 2233712640,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.14.attn_output.weight",
"offset": 2248458240,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.14.attn_q.weight",
"offset": 2263203840,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.14.attn_v.weight",
"offset": 2277949440,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.15.attn_k.weight",
"offset": 2292695040,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.15.attn_q.weight",
"offset": 2307440640,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.7.attn_norm.weight",
"offset": 2322186240,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 2322206720,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 2362019840,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.7.ffn_up.weight",
"offset": 2401832960,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 2441646080,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.8.attn_norm.weight",
"offset": 2441666560,
"shape": [
5120
],
"size": 20480,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 2441687040,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 2481500160,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.8.ffn_up.weight",
"offset": 2521313280,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 2561126400,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 2561146880,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.8.attn_output.weight",
"offset": 2575892480,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.8.attn_q.weight",
"offset": 2590638080,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.8.attn_v.weight",
"offset": 2605383680,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.9.attn_norm.weight",
"offset": 2620129280,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 2620149760,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 2659962880,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.9.ffn_up.weight",
"offset": 2699776000,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 2739589120,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 2739609600,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.9.attn_output.weight",
"offset": 2754355200,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.9.attn_q.weight",
"offset": 2769100800,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.9.attn_v.weight",
"offset": 2783846400,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.15.attn_norm.weight",
"offset": 2798592000,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 2798612480,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 2838425600,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.15.ffn_up.weight",
"offset": 2878238720,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 2918051840,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.15.attn_output.weight",
"offset": 2918072320,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.15.attn_v.weight",
"offset": 2932817920,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.16.attn_norm.weight",
"offset": 2947563520,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 2947584000,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 2987397120,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.16.ffn_up.weight",
"offset": 3027210240,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 3067023360,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 3067043840,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.16.attn_output.weight",
"offset": 3081789440,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.16.attn_q.weight",
"offset": 3096535040,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.16.attn_v.weight",
"offset": 3111280640,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.17.attn_norm.weight",
"offset": 3126026240,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 3126046720,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 3165859840,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.17.ffn_up.weight",
"offset": 3205672960,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 3245486080,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 3245506560,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.17.attn_output.weight",
"offset": 3260252160,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.17.attn_q.weight",
"offset": 3274997760,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.17.attn_v.weight",
"offset": 3289743360,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.18.attn_norm.weight",
"offset": 3304488960,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 3304509440,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 3344322560,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.18.ffn_up.weight",
"offset": 3384135680,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 3423948800,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 3423969280,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.18.attn_output.weight",
"offset": 3438714880,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.18.attn_q.weight",
"offset": 3453460480,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.18.attn_v.weight",
"offset": 3468206080,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.19.attn_norm.weight",
"offset": 3482951680,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 3482972160,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 3522785280,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.19.ffn_up.weight",
"offset": 3562598400,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 3602411520,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 3602432000,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.19.attn_output.weight",
"offset": 3617177600,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.19.attn_q.weight",
"offset": 3631923200,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.19.attn_v.weight",
"offset": 3646668800,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.20.attn_norm.weight",
"offset": 3661414400,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 3661434880,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 3701248000,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.20.ffn_up.weight",
"offset": 3741061120,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 3780874240,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 3780894720,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.20.attn_output.weight",
"offset": 3795640320,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.20.attn_q.weight",
"offset": 3810385920,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.20.attn_v.weight",
"offset": 3825131520,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.21.attn_norm.weight",
"offset": 3839877120,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 3839897600,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 3879710720,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.21.ffn_up.weight",
"offset": 3919523840,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 3959336960,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 3959357440,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.21.attn_output.weight",
"offset": 3974103040,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.21.attn_q.weight",
"offset": 3988848640,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.21.attn_v.weight",
"offset": 4003594240,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.22.attn_norm.weight",
"offset": 4018339840,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 4018360320,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 4058173440,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.22.ffn_up.weight",
"offset": 4097986560,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 4137799680,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 4137820160,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.22.attn_output.weight",
"offset": 4152565760,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.22.attn_q.weight",
"offset": 4167311360,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.22.attn_v.weight",
"offset": 4182056960,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.23.attn_norm.weight",
"offset": 4196802560,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 4196823040,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 4236636160,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.23.ffn_up.weight",
"offset": 4276449280,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 4316262400,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 4316282880,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.23.attn_output.weight",
"offset": 4331028480,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.23.attn_q.weight",
"offset": 4345774080,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.23.attn_v.weight",
"offset": 4360519680,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.24.attn_norm.weight",
"offset": 4375265280,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 4375285760,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 4415098880,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.24.ffn_up.weight",
"offset": 4454912000,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 4494725120,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 4494745600,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.24.attn_output.weight",
"offset": 4509491200,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.24.attn_q.weight",
"offset": 4524236800,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.24.attn_v.weight",
"offset": 4538982400,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.25.attn_norm.weight",
"offset": 4553728000,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 4553748480,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 4593561600,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.25.ffn_up.weight",
"offset": 4633374720,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 4673187840,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 4673208320,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.25.attn_output.weight",
"offset": 4687953920,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.25.attn_q.weight",
"offset": 4702699520,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.25.attn_v.weight",
"offset": 4717445120,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.26.attn_norm.weight",
"offset": 4732190720,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 4732211200,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 4772024320,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.26.ffn_up.weight",
"offset": 4811837440,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 4851650560,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 4851671040,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.26.attn_output.weight",
"offset": 4866416640,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.26.attn_q.weight",
"offset": 4881162240,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.26.attn_v.weight",
"offset": 4895907840,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.27.attn_norm.weight",
"offset": 4910653440,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 4910673920,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 4950487040,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.27.ffn_up.weight",
"offset": 4990300160,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 5030113280,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 5030133760,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.27.attn_output.weight",
"offset": 5044879360,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.27.attn_q.weight",
"offset": 5059624960,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.27.attn_v.weight",
"offset": 5074370560,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.28.attn_norm.weight",
"offset": 5089116160,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 5089136640,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 5128949760,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.28.ffn_up.weight",
"offset": 5168762880,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 5208576000,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 5208596480,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.28.attn_output.weight",
"offset": 5223342080,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.28.attn_q.weight",
"offset": 5238087680,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.28.attn_v.weight",
"offset": 5252833280,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.29.attn_norm.weight",
"offset": 5267578880,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 5267599360,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 5307412480,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.29.ffn_up.weight",
"offset": 5347225600,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 5387038720,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 5387059200,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.29.attn_output.weight",
"offset": 5401804800,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.29.attn_q.weight",
"offset": 5416550400,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.29.attn_v.weight",
"offset": 5431296000,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 5446041600,
"shape": [
5120,
13824
],
"size": 14745600,
"type": 2
},
{
"name": "blk.30.ffn_up.weight",
"offset": 5485854720,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.30.attn_k.weight",
"offset": 5525667840,
"shape": [
5120,
5120
],
"size": 39813120,
"type": 2
},
{
"name": "blk.30.attn_output.weight",
"offset": 5540413440,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.30.attn_q.weight",
"offset": 5555159040,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.30.attn_v.weight",
"offset": 5569904640,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.30.attn_norm.weight",
"offset": 5584650240,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 5584670720,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 5624483840,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.31.attn_norm.weight",
"offset": 5624504320,
"shape": [
5120
],
"size": 20480,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 5624524800,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 5664337920,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.31.ffn_up.weight",
"offset": 5704151040,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 5743964160,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 5743984640,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.31.attn_output.weight",
"offset": 5758730240,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.31.attn_q.weight",
"offset": 5773475840,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.31.attn_v.weight",
"offset": 5788221440,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.32.attn_norm.weight",
"offset": 5802967040,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 5802987520,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 5842800640,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.32.ffn_up.weight",
"offset": 5882613760,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 5922426880,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 5922447360,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.32.attn_output.weight",
"offset": 5937192960,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.32.attn_q.weight",
"offset": 5951938560,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.32.attn_v.weight",
"offset": 5966684160,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.33.attn_norm.weight",
"offset": 5981429760,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 5981450240,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 6021263360,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.33.ffn_up.weight",
"offset": 6061076480,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 6100889600,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.33.attn_k.weight",
"offset": 6100910080,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.33.attn_output.weight",
"offset": 6115655680,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.33.attn_q.weight",
"offset": 6130401280,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.33.attn_v.weight",
"offset": 6145146880,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.34.attn_norm.weight",
"offset": 6159892480,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 6159912960,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 6199726080,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.34.ffn_up.weight",
"offset": 6239539200,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 6279352320,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 6279372800,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.34.attn_output.weight",
"offset": 6294118400,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.34.attn_q.weight",
"offset": 6308864000,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.34.attn_v.weight",
"offset": 6323609600,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.35.attn_norm.weight",
"offset": 6338355200,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 6338375680,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 6378188800,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.35.ffn_up.weight",
"offset": 6418001920,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 6457815040,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 6457835520,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.35.attn_output.weight",
"offset": 6472581120,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.35.attn_q.weight",
"offset": 6487326720,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.35.attn_v.weight",
"offset": 6502072320,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.36.attn_norm.weight",
"offset": 6516817920,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 6516838400,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 6556651520,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.36.ffn_up.weight",
"offset": 6596464640,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 6636277760,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 6636298240,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.36.attn_output.weight",
"offset": 6651043840,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.36.attn_q.weight",
"offset": 6665789440,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.36.attn_v.weight",
"offset": 6680535040,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.37.attn_norm.weight",
"offset": 6695280640,
"shape": [
5120
],
"size": 14745600,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 6695301120,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 6735114240,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.37.ffn_up.weight",
"offset": 6774927360,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 6814740480,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 6814760960,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.37.attn_output.weight",
"offset": 6829506560,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.37.attn_q.weight",
"offset": 6844252160,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.37.attn_v.weight",
"offset": 6858997760,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 6873743360,
"shape": [
5120,
13824
],
"size": 14745600,
"type": 2
},
{
"name": "blk.38.attn_k.weight",
"offset": 6913556480,
"shape": [
5120,
5120
],
"size": 39813120,
"type": 2
},
{
"name": "blk.38.attn_output.weight",
"offset": 6928302080,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.38.attn_q.weight",
"offset": 6943047680,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.38.attn_v.weight",
"offset": 6957793280,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "output.weight",
"offset": 6972538880,
"shape": [
5120,
32000
],
"size": 14745600,
"type": 14
},
{
"name": "blk.38.attn_norm.weight",
"offset": 7106938880,
"shape": [
5120
],
"size": 134400000,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 7106959360,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.38.ffn_up.weight",
"offset": 7146772480,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 7186585600,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.39.attn_norm.weight",
"offset": 7186606080,
"shape": [
5120
],
"size": 20480,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 7186626560,
"shape": [
13824,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 7226439680,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.39.ffn_up.weight",
"offset": 7266252800,
"shape": [
5120,
13824
],
"size": 39813120,
"type": 2
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 7306065920,
"shape": [
5120
],
"size": 39813120,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 7306086400,
"shape": [
5120,
5120
],
"size": 20480,
"type": 2
},
{
"name": "blk.39.attn_output.weight",
"offset": 7320832000,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.39.attn_q.weight",
"offset": 7335577600,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "blk.39.attn_v.weight",
"offset": 7350323200,
"shape": [
5120,
5120
],
"size": 14745600,
"type": 2
},
{
"name": "output_norm.weight",
"offset": 7365068800,
"shape": [
5120
],
"size": 14745600,
"type": 0
}
],
"version": 3
}