latest
4.9GB
Introducing Emotional Llama, the model fine-tuned as an exercise for the live event on Ollama discord channer. Designed to understand and respond to a wide range of emotions.
13 Pulls Updated 8 days ago
245931d976c2 · 4.9GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 15,
"general.name": "emotional_llama_q4_k_m",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128255,
"tokenizer.ggml.pre": "llama-bpe",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 295501824,
"shape": [
4096
],
"size": 295501824,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 295518208,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 343687168,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.0.ffn_up.weight",
"offset": 376717312,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 409747456,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 409763840,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.0.attn_output.weight",
"offset": 412123136,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.0.attn_q.weight",
"offset": 421560320,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.0.attn_v.weight",
"offset": 430997504,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 434438144,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 434454528,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 482623488,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.1.ffn_up.weight",
"offset": 515653632,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 548683776,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 548700160,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.1.attn_output.weight",
"offset": 551059456,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.1.attn_q.weight",
"offset": 560496640,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.1.attn_v.weight",
"offset": 569933824,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 573374464,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 573390848,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 621559808,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.2.ffn_up.weight",
"offset": 654589952,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 687620096,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 687636480,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.2.attn_output.weight",
"offset": 689995776,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.2.attn_q.weight",
"offset": 699432960,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.2.attn_v.weight",
"offset": 708870144,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 712310784,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 712327168,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 760496128,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.3.ffn_up.weight",
"offset": 793526272,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 826556416,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 826572800,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.3.attn_output.weight",
"offset": 828932096,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.3.attn_q.weight",
"offset": 838369280,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.3.attn_v.weight",
"offset": 847806464,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 851247104,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 851263488,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 884293632,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.4.ffn_up.weight",
"offset": 917323776,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 950353920,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 950370304,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.4.attn_output.weight",
"offset": 952729600,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.4.attn_q.weight",
"offset": 962166784,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.4.attn_v.weight",
"offset": 971603968,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.5.attn_norm.weight",
"offset": 973963264,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 973979648,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1007009792,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1040039936,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1073070080,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1073086464,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.5.attn_output.weight",
"offset": 1075445760,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.5.attn_q.weight",
"offset": 1084882944,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.5.attn_v.weight",
"offset": 1094320128,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1096679424,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1096695808,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 1144864768,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.6.ffn_up.weight",
"offset": 1177894912,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 1210925056,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 1210941440,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.6.attn_output.weight",
"offset": 1213300736,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.6.attn_q.weight",
"offset": 1222737920,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.6.attn_v.weight",
"offset": 1232175104,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 1235615744,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 1235632128,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 1268662272,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.7.ffn_up.weight",
"offset": 1301692416,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 1334722560,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 1334738944,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.7.attn_output.weight",
"offset": 1337098240,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.7.attn_q.weight",
"offset": 1346535424,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.7.attn_v.weight",
"offset": 1355972608,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.8.attn_norm.weight",
"offset": 1358331904,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 1358348288,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 1391378432,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.8.ffn_up.weight",
"offset": 1424408576,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 1457438720,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 1457455104,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.8.attn_output.weight",
"offset": 1459814400,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.8.attn_q.weight",
"offset": 1469251584,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.8.attn_v.weight",
"offset": 1478688768,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.10.attn_norm.weight",
"offset": 1481048064,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 1481064448,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 1529233408,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.10.ffn_up.weight",
"offset": 1562263552,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 1595293696,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 1595310080,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.10.attn_output.weight",
"offset": 1597669376,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.10.attn_q.weight",
"offset": 1607106560,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.10.attn_v.weight",
"offset": 1616543744,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 1619984384,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 1620000768,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 1653030912,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.11.ffn_up.weight",
"offset": 1686061056,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 1719091200,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 1719107584,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.11.attn_output.weight",
"offset": 1721466880,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.11.attn_q.weight",
"offset": 1730904064,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.11.attn_v.weight",
"offset": 1740341248,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.12.attn_norm.weight",
"offset": 1742700544,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 1742716928,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 1775747072,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.12.ffn_up.weight",
"offset": 1808777216,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 1841807360,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 1841823744,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.12.attn_output.weight",
"offset": 1844183040,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.12.attn_q.weight",
"offset": 1853620224,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.12.attn_v.weight",
"offset": 1863057408,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.13.attn_norm.weight",
"offset": 1865416704,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 1865433088,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 1913602048,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.13.ffn_up.weight",
"offset": 1946632192,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 1979662336,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 1979678720,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.13.attn_output.weight",
"offset": 1982038016,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.13.attn_q.weight",
"offset": 1991475200,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.13.attn_v.weight",
"offset": 2000912384,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 2004353024,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 2004369408,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 2037399552,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.14.ffn_up.weight",
"offset": 2070429696,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 2103459840,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 2103476224,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.14.attn_output.weight",
"offset": 2105835520,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.14.attn_q.weight",
"offset": 2115272704,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.14.attn_v.weight",
"offset": 2124709888,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.15.attn_norm.weight",
"offset": 2127069184,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 2127085568,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 2160115712,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.15.ffn_up.weight",
"offset": 2193145856,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 2226176000,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 2226192384,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.15.attn_output.weight",
"offset": 2228551680,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.15.attn_q.weight",
"offset": 2237988864,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.15.attn_v.weight",
"offset": 2247426048,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.16.attn_norm.weight",
"offset": 2249785344,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 2249801728,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 2297970688,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.16.ffn_up.weight",
"offset": 2331000832,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 2364030976,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 2364047360,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.16.attn_output.weight",
"offset": 2366406656,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.16.attn_q.weight",
"offset": 2375843840,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.16.attn_v.weight",
"offset": 2385281024,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 2388721664,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 2388738048,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 2421768192,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.17.ffn_up.weight",
"offset": 2454798336,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 2487828480,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 2487844864,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.17.attn_output.weight",
"offset": 2490204160,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.17.attn_q.weight",
"offset": 2499641344,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.17.attn_v.weight",
"offset": 2509078528,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.18.attn_norm.weight",
"offset": 2511437824,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 2511454208,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 2544484352,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.18.ffn_up.weight",
"offset": 2577514496,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 2610544640,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 2610561024,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.18.attn_output.weight",
"offset": 2612920320,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.18.attn_q.weight",
"offset": 2622357504,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.18.attn_v.weight",
"offset": 2631794688,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.19.attn_norm.weight",
"offset": 2634153984,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 2634170368,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 2682339328,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.19.ffn_up.weight",
"offset": 2715369472,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 2748399616,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 2748416000,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.19.attn_output.weight",
"offset": 2750775296,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.19.attn_q.weight",
"offset": 2760212480,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.19.attn_v.weight",
"offset": 2769649664,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 2773090304,
"shape": [
4096,
14336
],
"size": 3440640,
"type": 12
},
{
"name": "blk.20.attn_k.weight",
"offset": 2806120448,
"shape": [
4096,
1024
],
"size": 33030144,
"type": 12
},
{
"name": "blk.20.attn_output.weight",
"offset": 2808479744,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.20.attn_q.weight",
"offset": 2817916928,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.20.attn_v.weight",
"offset": 2827354112,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.9.attn_norm.weight",
"offset": 2829713408,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 2829729792,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 2862759936,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.9.ffn_up.weight",
"offset": 2895790080,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 2928820224,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 2928836608,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.9.attn_output.weight",
"offset": 2931195904,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.9.attn_q.weight",
"offset": 2940633088,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.9.attn_v.weight",
"offset": 2950070272,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.20.attn_norm.weight",
"offset": 2952429568,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 2952445952,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.20.ffn_up.weight",
"offset": 2985476096,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 3018506240,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.21.attn_norm.weight",
"offset": 3018522624,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 3018539008,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 3066707968,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.21.ffn_up.weight",
"offset": 3099738112,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 3132768256,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 3132784640,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.21.attn_output.weight",
"offset": 3135143936,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.21.attn_q.weight",
"offset": 3144581120,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.21.attn_v.weight",
"offset": 3154018304,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 3157458944,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 3157475328,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 3190505472,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.22.ffn_up.weight",
"offset": 3223535616,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 3256565760,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 3256582144,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.22.attn_output.weight",
"offset": 3258941440,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.22.attn_q.weight",
"offset": 3268378624,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.22.attn_v.weight",
"offset": 3277815808,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.23.attn_norm.weight",
"offset": 3280175104,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 3280191488,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 3313221632,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.23.ffn_up.weight",
"offset": 3346251776,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 3379281920,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 3379298304,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.23.attn_output.weight",
"offset": 3381657600,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.23.attn_q.weight",
"offset": 3391094784,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.23.attn_v.weight",
"offset": 3400531968,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.24.attn_norm.weight",
"offset": 3402891264,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 3402907648,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 3451076608,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.24.ffn_up.weight",
"offset": 3484106752,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 3517136896,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 3517153280,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.24.attn_output.weight",
"offset": 3519512576,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.24.attn_q.weight",
"offset": 3528949760,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.24.attn_v.weight",
"offset": 3538386944,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 3541827584,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 3541843968,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 3574874112,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.25.ffn_up.weight",
"offset": 3607904256,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 3640934400,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 3640950784,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.25.attn_output.weight",
"offset": 3643310080,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.25.attn_q.weight",
"offset": 3652747264,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.25.attn_v.weight",
"offset": 3662184448,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.26.attn_norm.weight",
"offset": 3664543744,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 3664560128,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 3697590272,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.26.ffn_up.weight",
"offset": 3730620416,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 3763650560,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 3763666944,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.26.attn_output.weight",
"offset": 3766026240,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.26.attn_q.weight",
"offset": 3775463424,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.26.attn_v.weight",
"offset": 3784900608,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.27.attn_norm.weight",
"offset": 3787259904,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 3787276288,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 3835445248,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.27.ffn_up.weight",
"offset": 3868475392,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 3901505536,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 3901521920,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.27.attn_output.weight",
"offset": 3903881216,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.27.attn_q.weight",
"offset": 3913318400,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.27.attn_v.weight",
"offset": 3922755584,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 3926196224,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 3926212608,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 3974381568,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.28.ffn_up.weight",
"offset": 4007411712,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 4040441856,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 4040458240,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.28.attn_output.weight",
"offset": 4042817536,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.28.attn_q.weight",
"offset": 4052254720,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.28.attn_v.weight",
"offset": 4061691904,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 4065132544,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 4065148928,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 4113317888,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.29.ffn_up.weight",
"offset": 4146348032,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 4179378176,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 4179394560,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.29.attn_output.weight",
"offset": 4181753856,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.29.attn_q.weight",
"offset": 4191191040,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.29.attn_v.weight",
"offset": 4200628224,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 4204068864,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 4204085248,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 4252254208,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.30.ffn_up.weight",
"offset": 4285284352,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 4318314496,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 4318330880,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.30.attn_output.weight",
"offset": 4320690176,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.30.attn_q.weight",
"offset": 4330127360,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.30.attn_v.weight",
"offset": 4339564544,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 4343005184,
"shape": [
4096,
14336
],
"size": 3440640,
"type": 12
},
{
"name": "blk.31.ffn_up.weight",
"offset": 4376035328,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.31.attn_k.weight",
"offset": 4409065472,
"shape": [
4096,
1024
],
"size": 33030144,
"type": 12
},
{
"name": "blk.31.attn_output.weight",
"offset": 4411424768,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.31.attn_q.weight",
"offset": 4420861952,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.31.attn_v.weight",
"offset": 4430299136,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "output.weight",
"offset": 4433739776,
"shape": [
4096,
128256
],
"size": 3440640,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 4864679936,
"shape": [
4096
],
"size": 430940160,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 4864696320,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 4912865280,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 4912881664,
"shape": [
4096
],
"size": 16384,
"type": 0
}
],
"version": 3
}