Lexi is uncensored, which makes the model compliant. You are advised to implement your own alignment layer before exposing the model as a service. It will be highly compliant with any requests, even unethical ones.
8B
2,804 Pulls Updated 5 weeks ago
9a425d13c2dd · 8.5GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 7,
"general.name": "bk",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128009,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 558170112,
"shape": [
4096
],
"size": 558170112,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 558186496,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 620576768,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.0.ffn_up.weight",
"offset": 682967040,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 745357312,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 745373696,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.0.attn_output.weight",
"offset": 749830144,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.0.attn_q.weight",
"offset": 767655936,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "\u0000\u0000\u0000\u0000\u0000\u0000blk.18.attn_q",
"offset": 785481728,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.1.attn_norm.weight",
"offset": 789938176,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 789954560,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 852344832,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.1.ffn_up.weight",
"offset": 914735104,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 977125376,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 977141760,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.1.attn_output.weight",
"offset": 981598208,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.1.attn_q.weight",
"offset": 999424000,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.1.attn_v.weight",
"offset": 1017249792,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.2.attn_norm.weight",
"offset": 1021706240,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 1021722624,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 1084112896,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.2.ffn_up.weight",
"offset": 1146503168,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 1208893440,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 1208909824,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.2.attn_output.weight",
"offset": 1213366272,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.2.attn_q.weight",
"offset": 1231192064,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.2.attn_v.weight",
"offset": 1249017856,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.3.attn_norm.weight",
"offset": 1253474304,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 1253490688,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 1315880960,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.3.ffn_up.weight",
"offset": 1378271232,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 1440661504,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 1440677888,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.3.attn_output.weight",
"offset": 1445134336,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.3.attn_q.weight",
"offset": 1462960128,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.3.attn_v.weight",
"offset": 1480785920,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.4.attn_norm.weight",
"offset": 1485242368,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 1485258752,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 1547649024,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1610039296,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1672429568,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 1672445952,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.4.attn_output.weight",
"offset": 1676902400,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.4.attn_q.weight",
"offset": 1694728192,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.4.attn_v.weight",
"offset": 1712553984,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.5.attn_norm.weight",
"offset": 1717010432,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 1717026816,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1779417088,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1841807360,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1904197632,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1904214016,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.5.attn_output.weight",
"offset": 1908670464,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.5.attn_q.weight",
"offset": 1926496256,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.5.attn_v.weight",
"offset": 1944322048,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1948778496,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1948794880,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 2011185152,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.6.ffn_up.weight",
"offset": 2073575424,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 2135965696,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 2135982080,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.6.attn_output.weight",
"offset": 2140438528,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.6.attn_q.weight",
"offset": 2158264320,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.6.attn_v.weight",
"offset": 2176090112,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.7.attn_norm.weight",
"offset": 2180546560,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 2180562944,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 2242953216,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.7.ffn_up.weight",
"offset": 2305343488,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 2367733760,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 2367750144,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.7.attn_output.weight",
"offset": 2372206592,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.7.attn_q.weight",
"offset": 2390032384,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.7.attn_v.weight",
"offset": 2407858176,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.10.attn_norm.weight",
"offset": 2412314624,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 2412331008,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 2474721280,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.10.ffn_up.weight",
"offset": 2537111552,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 2599501824,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 2599518208,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.10.attn_output.weight",
"offset": 2603974656,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.10.attn_q.weight",
"offset": 2621800448,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.10.attn_v.weight",
"offset": 2639626240,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.11.attn_norm.weight",
"offset": 2644082688,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 2644099072,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 2706489344,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.11.ffn_up.weight",
"offset": 2768879616,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 2831269888,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 2831286272,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.11.attn_output.weight",
"offset": 2835742720,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.11.attn_q.weight",
"offset": 2853568512,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.11.attn_v.weight",
"offset": 2871394304,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.12.attn_norm.weight",
"offset": 2875850752,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 2875867136,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 2938257408,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.12.ffn_up.weight",
"offset": 3000647680,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 3063037952,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 3063054336,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.12.attn_output.weight",
"offset": 3067510784,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.12.attn_q.weight",
"offset": 3085336576,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.12.attn_v.weight",
"offset": 3103162368,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.13.attn_norm.weight",
"offset": 3107618816,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 3107635200,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 3170025472,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.13.ffn_up.weight",
"offset": 3232415744,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 3294806016,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 3294822400,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.13.attn_output.weight",
"offset": 3299278848,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.13.attn_q.weight",
"offset": 3317104640,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.13.attn_v.weight",
"offset": 3334930432,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.14.attn_norm.weight",
"offset": 3339386880,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 3339403264,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 3401793536,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.14.ffn_up.weight",
"offset": 3464183808,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 3526574080,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 3526590464,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.14.attn_output.weight",
"offset": 3531046912,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.14.attn_q.weight",
"offset": 3548872704,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.14.attn_v.weight",
"offset": 3566698496,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.15.attn_norm.weight",
"offset": 3571154944,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 3571171328,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 3633561600,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.15.ffn_up.weight",
"offset": 3695951872,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 3758342144,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 3758358528,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.15.attn_output.weight",
"offset": 3762814976,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.15.attn_q.weight",
"offset": 3780640768,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.15.attn_v.weight",
"offset": 3798466560,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.16.attn_norm.weight",
"offset": 3802923008,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 3802939392,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 3865329664,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.16.ffn_up.weight",
"offset": 3927719936,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 3990110208,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 3990126592,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.16.attn_output.weight",
"offset": 3994583040,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.16.attn_q.weight",
"offset": 4012408832,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.16.attn_v.weight",
"offset": 4030234624,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.17.attn_norm.weight",
"offset": 4034691072,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 4034707456,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 4097097728,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.17.ffn_up.weight",
"offset": 4159488000,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 4221878272,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 4221894656,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.17.attn_output.weight",
"offset": 4226351104,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.17.attn_q.weight",
"offset": 4244176896,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.17.attn_v.weight",
"offset": 4262002688,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.18.attn_k.weight",
"offset": 4266459136,
"shape": [
4096,
1024
],
"size": 4456448,
"type": 8
},
{
"name": "blk.18.attn_output.weight",
"offset": 4270915584,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.18.attn_q.weight",
"offset": 4288741376,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.18.attn_v.weight",
"offset": 4306567168,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.8.attn_norm.weight",
"offset": 4311023616,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 4311040000,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 4373430272,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.8.ffn_up.weight",
"offset": 4435820544,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 4498210816,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 4498227200,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.8.attn_output.weight",
"offset": 4502683648,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.8.attn_q.weight",
"offset": 4520509440,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.8.attn_v.weight",
"offset": 4538335232,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.9.attn_norm.weight",
"offset": 4542791680,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 4542808064,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 4605198336,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.9.ffn_up.weight",
"offset": 4667588608,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 4729978880,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 4729995264,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.9.attn_output.weight",
"offset": 4734451712,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.9.attn_q.weight",
"offset": 4752277504,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.9.attn_v.weight",
"offset": 4770103296,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.18.attn_norm.weight",
"offset": 4774559744,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 4774576128,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 4836966400,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.18.ffn_up.weight",
"offset": 4899356672,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 4961746944,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.19.attn_norm.weight",
"offset": 4961763328,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 4961779712,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 5024169984,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.19.ffn_up.weight",
"offset": 5086560256,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 5148950528,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 5148966912,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.19.attn_output.weight",
"offset": 5153423360,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.19.attn_q.weight",
"offset": 5171249152,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.19.attn_v.weight",
"offset": 5189074944,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.20.attn_norm.weight",
"offset": 5193531392,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 5193547776,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 5255938048,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.20.ffn_up.weight",
"offset": 5318328320,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 5380718592,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 5380734976,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.20.attn_output.weight",
"offset": 5385191424,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.20.attn_q.weight",
"offset": 5403017216,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.20.attn_v.weight",
"offset": 5420843008,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.21.attn_norm.weight",
"offset": 5425299456,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 5425315840,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 5487706112,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.21.ffn_up.weight",
"offset": 5550096384,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 5612486656,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 5612503040,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.21.attn_output.weight",
"offset": 5616959488,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.21.attn_q.weight",
"offset": 5634785280,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.21.attn_v.weight",
"offset": 5652611072,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.22.attn_norm.weight",
"offset": 5657067520,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 5657083904,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 5719474176,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.22.ffn_up.weight",
"offset": 5781864448,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 5844254720,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 5844271104,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.22.attn_output.weight",
"offset": 5848727552,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.22.attn_q.weight",
"offset": 5866553344,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.22.attn_v.weight",
"offset": 5884379136,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.23.attn_norm.weight",
"offset": 5888835584,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 5888851968,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 5951242240,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.23.ffn_up.weight",
"offset": 6013632512,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 6076022784,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 6076039168,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.23.attn_output.weight",
"offset": 6080495616,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.23.attn_q.weight",
"offset": 6098321408,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.23.attn_v.weight",
"offset": 6116147200,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.24.attn_norm.weight",
"offset": 6120603648,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 6120620032,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 6183010304,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.24.ffn_up.weight",
"offset": 6245400576,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 6307790848,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 6307807232,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.24.attn_output.weight",
"offset": 6312263680,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.24.attn_q.weight",
"offset": 6330089472,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.24.attn_v.weight",
"offset": 6347915264,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.25.attn_norm.weight",
"offset": 6352371712,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 6352388096,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 6414778368,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.25.ffn_up.weight",
"offset": 6477168640,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 6539558912,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 6539575296,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.25.attn_output.weight",
"offset": 6544031744,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.25.attn_q.weight",
"offset": 6561857536,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.25.attn_v.weight",
"offset": 6579683328,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.26.attn_norm.weight",
"offset": 6584139776,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 6584156160,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 6646546432,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.26.ffn_up.weight",
"offset": 6708936704,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 6771326976,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 6771343360,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.26.attn_output.weight",
"offset": 6775799808,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.26.attn_q.weight",
"offset": 6793625600,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.26.attn_v.weight",
"offset": 6811451392,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.27.attn_norm.weight",
"offset": 6815907840,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 6815924224,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 6878314496,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.27.ffn_up.weight",
"offset": 6940704768,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 7003095040,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 7003111424,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.27.attn_output.weight",
"offset": 7007567872,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.27.attn_q.weight",
"offset": 7025393664,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.27.attn_v.weight",
"offset": 7043219456,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 7047675904,
"shape": [
4096,
14336
],
"size": 4456448,
"type": 8
},
{
"name": "blk.28.attn_k.weight",
"offset": 7110066176,
"shape": [
4096,
1024
],
"size": 62390272,
"type": 8
},
{
"name": "blk.28.attn_output.weight",
"offset": 7114522624,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.28.attn_q.weight",
"offset": 7132348416,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.28.attn_v.weight",
"offset": 7150174208,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "output.weight",
"offset": 7154630656,
"shape": [
4096,
128256
],
"size": 4456448,
"type": 8
},
{
"name": "blk.28.attn_norm.weight",
"offset": 7712800768,
"shape": [
4096
],
"size": 558170112,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 7712817152,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.28.ffn_up.weight",
"offset": 7775207424,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 7837597696,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.29.attn_norm.weight",
"offset": 7837614080,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 7837630464,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 7900020736,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.29.ffn_up.weight",
"offset": 7962411008,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 8024801280,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 8024817664,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.29.attn_output.weight",
"offset": 8029274112,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.29.attn_q.weight",
"offset": 8047099904,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.29.attn_v.weight",
"offset": 8064925696,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.30.attn_norm.weight",
"offset": 8069382144,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 8069398528,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 8131788800,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.30.ffn_up.weight",
"offset": 8194179072,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 8256569344,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 8256585728,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.30.attn_output.weight",
"offset": 8261042176,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.30.attn_q.weight",
"offset": 8278867968,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.30.attn_v.weight",
"offset": 8296693760,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.31.attn_norm.weight",
"offset": 8301150208,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 8301166592,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 8363556864,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.31.ffn_up.weight",
"offset": 8425947136,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 8488337408,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 8488353792,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.31.attn_output.weight",
"offset": 8492810240,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.31.attn_q.weight",
"offset": 8510636032,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.31.attn_v.weight",
"offset": 8528461824,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "output_norm.weight",
"offset": 8532918272,
"shape": [
4096
],
"size": 4456448,
"type": 0
}
],
"version": 3
}