Lexi is uncensored, which makes the model compliant. You are advised to implement your own alignment layer before exposing the model as a service. It will be highly compliant with any requests, even unethical ones.
8B
2,804 Pulls Updated 5 weeks ago
3f3a13eb3fbc · 4.9GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 15,
"general.name": "bk",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128009,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 295501824,
"shape": [
4096
],
"size": 295501824,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 295518208,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 343687168,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.0.ffn_up.weight",
"offset": 376717312,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 409747456,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 409763840,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.0.attn_output.weight",
"offset": 412123136,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.0.attn_q.weight",
"offset": 421560320,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.0.attn_v.weight",
"offset": 430997504,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 434438144,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 434454528,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 482623488,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.1.ffn_up.weight",
"offset": 515653632,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 548683776,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 548700160,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.1.attn_output.weight",
"offset": 551059456,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.1.attn_q.weight",
"offset": 560496640,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.1.attn_v.weight",
"offset": 569933824,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 573374464,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 573390848,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 621559808,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.2.ffn_up.weight",
"offset": 654589952,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 687620096,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 687636480,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.2.attn_output.weight",
"offset": 689995776,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.2.attn_q.weight",
"offset": 699432960,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.2.attn_v.weight",
"offset": 708870144,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 712310784,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 712327168,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 760496128,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.3.ffn_up.weight",
"offset": 793526272,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 826556416,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 826572800,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.3.attn_output.weight",
"offset": 828932096,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.3.attn_q.weight",
"offset": 838369280,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.3.attn_v.weight",
"offset": 847806464,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 851247104,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 851263488,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 884293632,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.4.ffn_up.weight",
"offset": 917323776,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 950353920,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 950370304,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.4.attn_output.weight",
"offset": 952729600,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.4.attn_q.weight",
"offset": 962166784,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.4.attn_v.weight",
"offset": 971603968,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.5.attn_norm.weight",
"offset": 973963264,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 973979648,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1007009792,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1040039936,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1073070080,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1073086464,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.5.attn_output.weight",
"offset": 1075445760,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.5.attn_q.weight",
"offset": 1084882944,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.5.attn_v.weight",
"offset": 1094320128,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1096679424,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1096695808,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 1144864768,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.6.ffn_up.weight",
"offset": 1177894912,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 1210925056,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 1210941440,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.6.attn_output.weight",
"offset": 1213300736,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.6.attn_q.weight",
"offset": 1222737920,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.6.attn_v.weight",
"offset": 1232175104,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 1235615744,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 1235632128,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 1268662272,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.7.ffn_up.weight",
"offset": 1301692416,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 1334722560,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 1334738944,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.7.attn_output.weight",
"offset": 1337098240,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.7.attn_q.weight",
"offset": 1346535424,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.7.attn_v.weight",
"offset": 1355972608,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.10.attn_norm.weight",
"offset": 1358331904,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 1358348288,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 1391378432,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.10.ffn_up.weight",
"offset": 1424408576,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 1457438720,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 1457455104,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.10.attn_output.weight",
"offset": 1459814400,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.10.attn_q.weight",
"offset": 1469251584,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.10.attn_v.weight",
"offset": 1478688768,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.11.attn_norm.weight",
"offset": 1481048064,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 1481064448,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 1529233408,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.11.ffn_up.weight",
"offset": 1562263552,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 1595293696,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 1595310080,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.11.attn_output.weight",
"offset": 1597669376,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.11.attn_q.weight",
"offset": 1607106560,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.11.attn_v.weight",
"offset": 1616543744,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 1619984384,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 1620000768,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 1653030912,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.12.ffn_up.weight",
"offset": 1686061056,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 1719091200,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 1719107584,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.12.attn_output.weight",
"offset": 1721466880,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.12.attn_q.weight",
"offset": 1730904064,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.12.attn_v.weight",
"offset": 1740341248,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.13.attn_norm.weight",
"offset": 1742700544,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 1742716928,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 1775747072,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.13.ffn_up.weight",
"offset": 1808777216,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 1841807360,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 1841823744,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.13.attn_output.weight",
"offset": 1844183040,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.13.attn_q.weight",
"offset": 1853620224,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.13.attn_v.weight",
"offset": 1863057408,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.14.attn_norm.weight",
"offset": 1865416704,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 1865433088,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 1913602048,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.14.ffn_up.weight",
"offset": 1946632192,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 1979662336,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 1979678720,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.14.attn_output.weight",
"offset": 1982038016,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.14.attn_q.weight",
"offset": 1991475200,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.14.attn_v.weight",
"offset": 2000912384,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 2004353024,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 2004369408,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 2037399552,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.15.ffn_up.weight",
"offset": 2070429696,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 2103459840,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 2103476224,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.15.attn_output.weight",
"offset": 2105835520,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.15.attn_q.weight",
"offset": 2115272704,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.15.attn_v.weight",
"offset": 2124709888,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.16.attn_norm.weight",
"offset": 2127069184,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 2127085568,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 2160115712,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.16.ffn_up.weight",
"offset": 2193145856,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 2226176000,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 2226192384,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.16.attn_output.weight",
"offset": 2228551680,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.16.attn_q.weight",
"offset": 2237988864,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.16.attn_v.weight",
"offset": 2247426048,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.17.attn_norm.weight",
"offset": 2249785344,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 2249801728,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 2297970688,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.17.ffn_up.weight",
"offset": 2331000832,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 2364030976,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 2364047360,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.17.attn_output.weight",
"offset": 2366406656,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.17.attn_q.weight",
"offset": 2375843840,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.17.attn_v.weight",
"offset": 2385281024,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.18.attn_k.weight",
"offset": 2388721664,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 12
},
{
"name": "\u0000\u0000\u0000\u0000\u0000\u00008\u0000\u0000\u0000\u0000\u0000\u0000\f\u0000\u0000\u0000\u0000\ufffdk\u001f\u0001\u0000\u0000\u0000",
"offset": 2391080960,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.18.attn_q.weight",
"offset": 2400518144,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.18.attn_v.weight",
"offset": 2409955328,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.8.attn_norm.weight",
"offset": 2412314624,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 2412331008,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 2445361152,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.8.ffn_up.weight",
"offset": 2478391296,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 2511421440,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 2511437824,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.8.attn_output.weight",
"offset": 2513797120,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.8.attn_q.weight",
"offset": 2523234304,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.8.attn_v.weight",
"offset": 2532671488,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.9.attn_norm.weight",
"offset": 2535030784,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 2535047168,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 2568077312,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.9.ffn_up.weight",
"offset": 2601107456,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 2634137600,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 2634153984,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.9.attn_output.weight",
"offset": 2636513280,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.9.attn_q.weight",
"offset": 2645950464,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.9.attn_v.weight",
"offset": 2655387648,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 2658828288,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 2658844672,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 2707013632,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.18.ffn_up.weight",
"offset": 2740043776,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 2773073920,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.19.attn_norm.weight",
"offset": 2773090304,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 2773106688,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 2806136832,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.19.ffn_up.weight",
"offset": 2839166976,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 2872197120,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 2872213504,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.19.attn_output.weight",
"offset": 2874572800,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.19.attn_q.weight",
"offset": 2884009984,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.19.attn_v.weight",
"offset": 2893447168,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.20.attn_norm.weight",
"offset": 2895806464,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 2895822848,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 2928852992,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.20.ffn_up.weight",
"offset": 2961883136,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 2994913280,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 2994929664,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.20.attn_output.weight",
"offset": 2997288960,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.20.attn_q.weight",
"offset": 3006726144,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.20.attn_v.weight",
"offset": 3016163328,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.21.attn_norm.weight",
"offset": 3018522624,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 3018539008,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 3066707968,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.21.ffn_up.weight",
"offset": 3099738112,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 3132768256,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 3132784640,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.21.attn_output.weight",
"offset": 3135143936,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.21.attn_q.weight",
"offset": 3144581120,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.21.attn_v.weight",
"offset": 3154018304,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 3157458944,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 3157475328,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 3190505472,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.22.ffn_up.weight",
"offset": 3223535616,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 3256565760,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 3256582144,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.22.attn_output.weight",
"offset": 3258941440,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.22.attn_q.weight",
"offset": 3268378624,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.22.attn_v.weight",
"offset": 3277815808,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.23.attn_norm.weight",
"offset": 3280175104,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 3280191488,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 3313221632,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.23.ffn_up.weight",
"offset": 3346251776,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 3379281920,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 3379298304,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.23.attn_output.weight",
"offset": 3381657600,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.23.attn_q.weight",
"offset": 3391094784,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.23.attn_v.weight",
"offset": 3400531968,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.24.attn_norm.weight",
"offset": 3402891264,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 3402907648,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 3451076608,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.24.ffn_up.weight",
"offset": 3484106752,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 3517136896,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 3517153280,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.24.attn_output.weight",
"offset": 3519512576,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.24.attn_q.weight",
"offset": 3528949760,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.24.attn_v.weight",
"offset": 3538386944,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 3541827584,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 3541843968,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 3574874112,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.25.ffn_up.weight",
"offset": 3607904256,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 3640934400,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 3640950784,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.25.attn_output.weight",
"offset": 3643310080,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.25.attn_q.weight",
"offset": 3652747264,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.25.attn_v.weight",
"offset": 3662184448,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.26.attn_norm.weight",
"offset": 3664543744,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 3664560128,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 3697590272,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.26.ffn_up.weight",
"offset": 3730620416,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 3763650560,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 3763666944,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.26.attn_output.weight",
"offset": 3766026240,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.26.attn_q.weight",
"offset": 3775463424,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.26.attn_v.weight",
"offset": 3784900608,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.27.attn_norm.weight",
"offset": 3787259904,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 3787276288,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 3835445248,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.27.ffn_up.weight",
"offset": 3868475392,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 3901505536,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 3901521920,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.27.attn_output.weight",
"offset": 3903881216,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.27.attn_q.weight",
"offset": 3913318400,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.27.attn_v.weight",
"offset": 3922755584,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 3926196224,
"shape": [
4096,
14336
],
"size": 3440640,
"type": 12
},
{
"name": "blk.28.attn_k.weight",
"offset": 3959226368,
"shape": [
4096,
1024
],
"size": 33030144,
"type": 12
},
{
"name": "blk.28.attn_output.weight",
"offset": 3961585664,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.28.attn_q.weight",
"offset": 3971022848,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.28.attn_v.weight",
"offset": 3980460032,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "output.weight",
"offset": 3983900672,
"shape": [
4096,
128256
],
"size": 3440640,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 4414840832,
"shape": [
4096
],
"size": 430940160,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 4414857216,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.ffn_up.weight",
"offset": 4463026176,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 4496056320,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.29.attn_norm.weight",
"offset": 4496072704,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 4496089088,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 4544258048,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.29.ffn_up.weight",
"offset": 4577288192,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 4610318336,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 4610334720,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.29.attn_output.weight",
"offset": 4612694016,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.29.attn_q.weight",
"offset": 4622131200,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.29.attn_v.weight",
"offset": 4631568384,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 4635009024,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 4635025408,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 4683194368,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.30.ffn_up.weight",
"offset": 4716224512,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 4749254656,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 4749271040,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.30.attn_output.weight",
"offset": 4751630336,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.30.attn_q.weight",
"offset": 4761067520,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.30.attn_v.weight",
"offset": 4770504704,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 4773945344,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 4773961728,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 4822130688,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.31.ffn_up.weight",
"offset": 4855160832,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 4888190976,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "\u0000\u0000\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\f\u0000\u0000\u0000\u0000 \u0010$\u0001",
"offset": 4888207360,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.31.attn_output.weight",
"offset": 4890566656,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.31.attn_q.weight",
"offset": 4900003840,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.31.attn_v.weight",
"offset": 4909441024,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "output_norm.weight",
"offset": 4912881664,
"shape": [
4096
],
"size": 3440640,
"type": 0
}
],
"version": 3
}