Lexi is uncensored, which makes the model compliant. You are advised to implement your own alignment layer before exposing the model as a service. It will be highly compliant with any requests, even unethical ones.
8B
2,804 Pulls Updated 5 weeks ago
8b25788328bf · 5.7GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 17,
"general.name": "bk",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128009,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 361168896,
"shape": [
4096
],
"size": 361168896,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 361185280,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 409354240,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.0.ffn_up.weight",
"offset": 449724416,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 490094592,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 490110976,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.0.attn_output.weight",
"offset": 492994560,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.0.attn_q.weight",
"offset": 504528896,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.0.attn_v.weight",
"offset": 516063232,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 519503872,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 519520256,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 567689216,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.1.ffn_up.weight",
"offset": 608059392,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 648429568,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 648445952,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.1.attn_output.weight",
"offset": 651329536,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.1.attn_q.weight",
"offset": 662863872,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.1.attn_v.weight",
"offset": 674398208,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 677838848,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 677855232,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 726024192,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.2.ffn_up.weight",
"offset": 766394368,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 806764544,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 806780928,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.2.attn_output.weight",
"offset": 809664512,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.2.attn_q.weight",
"offset": 821198848,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.2.attn_v.weight",
"offset": 832733184,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 836173824,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 836190208,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 884359168,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.3.ffn_up.weight",
"offset": 924729344,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 965099520,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 965115904,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.3.attn_output.weight",
"offset": 967999488,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.3.attn_q.weight",
"offset": 979533824,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.3.attn_v.weight",
"offset": 991068160,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 994508800,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 994525184,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 1034895360,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1075265536,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1115635712,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 1115652096,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.4.attn_output.weight",
"offset": 1118535680,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.4.attn_q.weight",
"offset": 1130070016,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.4.attn_v.weight",
"offset": 1141604352,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.5.attn_norm.weight",
"offset": 1144487936,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 1144504320,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1184874496,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1225244672,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1265614848,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1265631232,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.5.attn_output.weight",
"offset": 1268514816,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.5.attn_q.weight",
"offset": 1280049152,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.5.attn_v.weight",
"offset": 1291583488,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1294467072,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1294483456,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 1342652416,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.6.ffn_up.weight",
"offset": 1383022592,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 1423392768,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 1423409152,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.6.attn_output.weight",
"offset": 1426292736,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.6.attn_q.weight",
"offset": 1437827072,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.6.attn_v.weight",
"offset": 1449361408,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 1452802048,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 1452818432,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 1493188608,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.7.ffn_up.weight",
"offset": 1533558784,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 1573928960,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 1573945344,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.7.attn_output.weight",
"offset": 1576828928,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.7.attn_q.weight",
"offset": 1588363264,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.7.attn_v.weight",
"offset": 1599897600,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.10.attn_norm.weight",
"offset": 1602781184,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 1602797568,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 1643167744,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.10.ffn_up.weight",
"offset": 1683537920,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 1723908096,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 1723924480,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.10.attn_output.weight",
"offset": 1726808064,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.10.attn_q.weight",
"offset": 1738342400,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.10.attn_v.weight",
"offset": 1749876736,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.11.attn_norm.weight",
"offset": 1752760320,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 1752776704,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 1800945664,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.11.ffn_up.weight",
"offset": 1841315840,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 1881686016,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 1881702400,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.11.attn_output.weight",
"offset": 1884585984,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.11.attn_q.weight",
"offset": 1896120320,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.11.attn_v.weight",
"offset": 1907654656,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 1911095296,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 1911111680,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 1951481856,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.12.ffn_up.weight",
"offset": 1991852032,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 2032222208,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 2032238592,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.12.attn_output.weight",
"offset": 2035122176,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.12.attn_q.weight",
"offset": 2046656512,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.12.attn_v.weight",
"offset": 2058190848,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.13.attn_norm.weight",
"offset": 2061074432,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 2061090816,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 2101460992,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.13.ffn_up.weight",
"offset": 2141831168,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 2182201344,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 2182217728,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.13.attn_output.weight",
"offset": 2185101312,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.13.attn_q.weight",
"offset": 2196635648,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.13.attn_v.weight",
"offset": 2208169984,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.14.attn_norm.weight",
"offset": 2211053568,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 2211069952,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 2259238912,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.14.ffn_up.weight",
"offset": 2299609088,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 2339979264,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 2339995648,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.14.attn_output.weight",
"offset": 2342879232,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.14.attn_q.weight",
"offset": 2354413568,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.14.attn_v.weight",
"offset": 2365947904,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 2369388544,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 2369404928,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 2409775104,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.15.ffn_up.weight",
"offset": 2450145280,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 2490515456,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 2490531840,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.15.attn_output.weight",
"offset": 2493415424,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.15.attn_q.weight",
"offset": 2504949760,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.15.attn_v.weight",
"offset": 2516484096,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.16.attn_norm.weight",
"offset": 2519367680,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 2519384064,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 2559754240,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.16.ffn_up.weight",
"offset": 2600124416,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 2640494592,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 2640510976,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.16.attn_output.weight",
"offset": 2643394560,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.16.attn_q.weight",
"offset": 2654928896,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.16.attn_v.weight",
"offset": 2666463232,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.17.attn_norm.weight",
"offset": 2669346816,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 2669363200,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 2717532160,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.17.ffn_up.weight",
"offset": 2757902336,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 2798272512,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 2798288896,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.17.attn_output.weight",
"offset": 2801172480,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.17.attn_q.weight",
"offset": 2812706816,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.17.attn_v.weight",
"offset": 2824241152,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.18.attn_k.weight",
"offset": 2827681792,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 13
},
{
"name": "blk.18.attn_output.weight",
"offset": 2830565376,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.18.attn_q.weight",
"offset": 2842099712,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.18.attn_v.weight",
"offset": 2853634048,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.8.attn_norm.weight",
"offset": 2856517632,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 2856534016,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 2896904192,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.8.ffn_up.weight",
"offset": 2937274368,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 2977644544,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 2977660928,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.8.attn_output.weight",
"offset": 2980544512,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.8.attn_q.weight",
"offset": 2992078848,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.8.attn_v.weight",
"offset": 3003613184,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.9.attn_norm.weight",
"offset": 3006496768,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 3006513152,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 3046883328,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.9.ffn_up.weight",
"offset": 3087253504,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 3127623680,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 3127640064,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.9.attn_output.weight",
"offset": 3130523648,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.9.attn_q.weight",
"offset": 3142057984,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.9.attn_v.weight",
"offset": 3153592320,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 3157032960,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 3157049344,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 3205218304,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.18.ffn_up.weight",
"offset": 3245588480,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 3285958656,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.19.attn_norm.weight",
"offset": 3285975040,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 3285991424,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 3326361600,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.19.ffn_up.weight",
"offset": 3366731776,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 3407101952,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 3407118336,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.19.attn_output.weight",
"offset": 3410001920,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.19.attn_q.weight",
"offset": 3421536256,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.19.attn_v.weight",
"offset": 3433070592,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.20.attn_norm.weight",
"offset": 3435954176,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 3435970560,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 3476340736,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.20.ffn_up.weight",
"offset": 3516710912,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 3557081088,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 3557097472,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.20.attn_output.weight",
"offset": 3559981056,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.20.attn_q.weight",
"offset": 3571515392,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.20.attn_v.weight",
"offset": 3583049728,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.21.attn_norm.weight",
"offset": 3585933312,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 3585949696,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 3634118656,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.21.ffn_up.weight",
"offset": 3674488832,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 3714859008,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 3714875392,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.21.attn_output.weight",
"offset": 3717758976,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.21.attn_q.weight",
"offset": 3729293312,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.21.attn_v.weight",
"offset": 3740827648,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 3744268288,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 3744284672,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 3784654848,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.22.ffn_up.weight",
"offset": 3825025024,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 3865395200,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 3865411584,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.22.attn_output.weight",
"offset": 3868295168,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.22.attn_q.weight",
"offset": 3879829504,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.22.attn_v.weight",
"offset": 3891363840,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.23.attn_norm.weight",
"offset": 3894247424,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 3894263808,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 3934633984,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.23.ffn_up.weight",
"offset": 3975004160,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 4015374336,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 4015390720,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.23.attn_output.weight",
"offset": 4018274304,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.23.attn_q.weight",
"offset": 4029808640,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.23.attn_v.weight",
"offset": 4041342976,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.24.attn_norm.weight",
"offset": 4044226560,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 4044242944,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 4092411904,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.24.ffn_up.weight",
"offset": 4132782080,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 4173152256,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 4173168640,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.24.attn_output.weight",
"offset": 4176052224,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.24.attn_q.weight",
"offset": 4187586560,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.24.attn_v.weight",
"offset": 4199120896,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 4202561536,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 4202577920,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 4242948096,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.25.ffn_up.weight",
"offset": 4283318272,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 4323688448,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 4323704832,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.25.attn_output.weight",
"offset": 4326588416,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.25.attn_q.weight",
"offset": 4338122752,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.25.attn_v.weight",
"offset": 4349657088,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.26.attn_norm.weight",
"offset": 4352540672,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 4352557056,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 4392927232,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.26.ffn_up.weight",
"offset": 4433297408,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 4473667584,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 4473683968,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.26.attn_output.weight",
"offset": 4476567552,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.26.attn_q.weight",
"offset": 4488101888,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.26.attn_v.weight",
"offset": 4499636224,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.27.attn_norm.weight",
"offset": 4502519808,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 4502536192,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 4550705152,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.27.ffn_up.weight",
"offset": 4591075328,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 4631445504,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 4631461888,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.27.attn_output.weight",
"offset": 4634345472,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.27.attn_q.weight",
"offset": 4645879808,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.27.attn_v.weight",
"offset": 4657414144,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 4660854784,
"shape": [
4096,
14336
],
"size": 3440640,
"type": 13
},
{
"name": "blk.28.attn_k.weight",
"offset": 4701224960,
"shape": [
4096,
1024
],
"size": 40370176,
"type": 13
},
{
"name": "blk.28.attn_output.weight",
"offset": 4704108544,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.28.attn_q.weight",
"offset": 4715642880,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.28.attn_v.weight",
"offset": 4727177216,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "output.weight",
"offset": 4730617856,
"shape": [
4096,
128256
],
"size": 3440640,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 5161558016,
"shape": [
4096
],
"size": 430940160,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 5161574400,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.ffn_up.weight",
"offset": 5209743360,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 5250113536,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.29.attn_norm.weight",
"offset": 5250129920,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 5250146304,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 5298315264,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.29.ffn_up.weight",
"offset": 5338685440,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 5379055616,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 5379072000,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.29.attn_output.weight",
"offset": 5381955584,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.29.attn_q.weight",
"offset": 5393489920,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.29.attn_v.weight",
"offset": 5405024256,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 5408464896,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 5408481280,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 5456650240,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.30.ffn_up.weight",
"offset": 5497020416,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 5537390592,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 5537406976,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.30.attn_output.weight",
"offset": 5540290560,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.30.attn_q.weight",
"offset": 5551824896,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.30.attn_v.weight",
"offset": 5563359232,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 5566799872,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 5566816256,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 5614985216,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.31.ffn_up.weight",
"offset": 5655355392,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 5695725568,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 5695741952,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.31.attn_output.weight",
"offset": 5698625536,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.31.attn_q.weight",
"offset": 5710159872,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.31.attn_v.weight",
"offset": 5721694208,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "output_norm.weight",
"offset": 5725134848,
"shape": [
4096
],
"size": 3440640,
"type": 0
}
],
"version": 3
}