Lexi is uncensored, which makes the model compliant. You are advised to implement your own alignment layer before exposing the model as a service. It will be highly compliant with any requests, even unethical ones.
8B
2,804 Pulls Updated 5 weeks ago
67cd8b5100d3 · 5.7GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 17,
"general.name": "LexiFun",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 361168896,
"shape": [
4096
],
"size": 361168896,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 361185280,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 409354240,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.0.ffn_up.weight",
"offset": 449724416,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 490094592,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 490110976,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.0.attn_output.weight",
"offset": 492994560,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.0.attn_q.weight",
"offset": 504528896,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.0.attn_v.weight",
"offset": 516063232,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 519503872,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 519520256,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 567689216,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.1.ffn_up.weight",
"offset": 608059392,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 648429568,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 648445952,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.1.attn_output.weight",
"offset": 651329536,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.1.attn_q.weight",
"offset": 662863872,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.1.attn_v.weight",
"offset": 674398208,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 677838848,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 677855232,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 726024192,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.2.ffn_up.weight",
"offset": 766394368,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 806764544,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 806780928,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.2.attn_output.weight",
"offset": 809664512,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.2.attn_q.weight",
"offset": 821198848,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.2.attn_v.weight",
"offset": 832733184,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 836173824,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 836190208,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 884359168,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.3.ffn_up.weight",
"offset": 924729344,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 965099520,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 965115904,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.3.attn_output.weight",
"offset": 967999488,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.3.attn_q.weight",
"offset": 979533824,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.3.attn_v.weight",
"offset": 991068160,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 994508800,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 994525184,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 1034895360,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1075265536,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1115635712,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 1115652096,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.4.attn_output.weight",
"offset": 1118535680,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.4.attn_q.weight",
"offset": 1130070016,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.4.attn_v.weight",
"offset": 1141604352,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.5.attn_norm.weight",
"offset": 1144487936,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 1144504320,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1184874496,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1225244672,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1265614848,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1265631232,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.5.attn_output.weight",
"offset": 1268514816,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.5.attn_q.weight",
"offset": 1280049152,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.5.attn_v.weight",
"offset": 1291583488,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1294467072,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1294483456,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 1342652416,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.6.ffn_up.weight",
"offset": 1383022592,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 1423392768,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 1423409152,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.6.attn_output.weight",
"offset": 1426292736,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.6.attn_q.weight",
"offset": 1437827072,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.6.attn_v.weight",
"offset": 1449361408,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 1452802048,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 1452818432,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 1493188608,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.7.ffn_up.weight",
"offset": 1533558784,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 1573928960,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 1573945344,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.7.attn_output.weight",
"offset": 1576828928,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.7.attn_q.weight",
"offset": 1588363264,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.7.attn_v.weight",
"offset": 1599897600,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.8.attn_norm.weight",
"offset": 1602781184,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 1602797568,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 1643167744,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.8.ffn_up.weight",
"offset": 1683537920,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 1723908096,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 1723924480,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.8.attn_output.weight",
"offset": 1726808064,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.8.attn_q.weight",
"offset": 1738342400,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.8.attn_v.weight",
"offset": 1749876736,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.10.attn_norm.weight",
"offset": 1752760320,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 1752776704,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 1800945664,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.10.ffn_up.weight",
"offset": 1841315840,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 1881686016,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 1881702400,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.10.attn_output.weight",
"offset": 1884585984,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.10.attn_q.weight",
"offset": 1896120320,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.10.attn_v.weight",
"offset": 1907654656,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 1911095296,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 1911111680,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 1951481856,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.11.ffn_up.weight",
"offset": 1991852032,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 2032222208,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 2032238592,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.11.attn_output.weight",
"offset": 2035122176,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.11.attn_q.weight",
"offset": 2046656512,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.11.attn_v.weight",
"offset": 2058190848,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.12.attn_norm.weight",
"offset": 2061074432,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 2061090816,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 2101460992,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.12.ffn_up.weight",
"offset": 2141831168,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 2182201344,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 2182217728,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.12.attn_output.weight",
"offset": 2185101312,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.12.attn_q.weight",
"offset": 2196635648,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.12.attn_v.weight",
"offset": 2208169984,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.13.attn_norm.weight",
"offset": 2211053568,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 2211069952,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 2259238912,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.13.ffn_up.weight",
"offset": 2299609088,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 2339979264,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 2339995648,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.13.attn_output.weight",
"offset": 2342879232,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.13.attn_q.weight",
"offset": 2354413568,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.13.attn_v.weight",
"offset": 2365947904,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 2369388544,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 2369404928,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 2409775104,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.14.ffn_up.weight",
"offset": 2450145280,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 2490515456,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 2490531840,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.14.attn_output.weight",
"offset": 2493415424,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.14.attn_q.weight",
"offset": 2504949760,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.14.attn_v.weight",
"offset": 2516484096,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.15.attn_norm.weight",
"offset": 2519367680,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 2519384064,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 2559754240,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.15.ffn_up.weight",
"offset": 2600124416,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 2640494592,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 2640510976,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.15.attn_output.weight",
"offset": 2643394560,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.15.attn_q.weight",
"offset": 2654928896,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.15.attn_v.weight",
"offset": 2666463232,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.16.attn_norm.weight",
"offset": 2669346816,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 2669363200,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 2717532160,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.16.ffn_up.weight",
"offset": 2757902336,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 2798272512,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 2798288896,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.16.attn_output.weight",
"offset": 2801172480,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.16.attn_q.weight",
"offset": 2812706816,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.16.attn_v.weight",
"offset": 2824241152,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 2827681792,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "\u0000\u0000blk.31.attn_output.w",
"offset": 2827698176,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 2868068352,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.17.ffn_up.weight",
"offset": 2908438528,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 2948808704,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 2948825088,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.17.attn_output.weight",
"offset": 2951708672,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.17.attn_q.weight",
"offset": 2963243008,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.17.attn_v.weight",
"offset": 2974777344,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.18.attn_norm.weight",
"offset": 2977660928,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 2977677312,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 3018047488,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.18.ffn_up.weight",
"offset": 3058417664,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 3098787840,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 3098804224,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.18.attn_output.weight",
"offset": 3101687808,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.18.attn_q.weight",
"offset": 3113222144,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.18.attn_v.weight",
"offset": 3124756480,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.19.attn_norm.weight",
"offset": 3127640064,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 3127656448,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 3175825408,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.19.ffn_up.weight",
"offset": 3216195584,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 3256565760,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 3256582144,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.19.attn_output.weight",
"offset": 3259465728,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.19.attn_q.weight",
"offset": 3271000064,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.19.attn_v.weight",
"offset": 3282534400,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 3285975040,
"shape": [
4096,
14336
],
"size": 3440640,
"type": 13
},
{
"name": "blk.20.attn_k.weight",
"offset": 3326345216,
"shape": [
4096,
1024
],
"size": 40370176,
"type": 13
},
{
"name": "blk.20.attn_output.weight",
"offset": 3329228800,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.20.attn_q.weight",
"offset": 3340763136,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.20.attn_v.weight",
"offset": 3352297472,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.9.attn_norm.weight",
"offset": 3355181056,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 3355197440,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 3395567616,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.9.ffn_up.weight",
"offset": 3435937792,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 3476307968,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 3476324352,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.9.attn_output.weight",
"offset": 3479207936,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.9.attn_q.weight",
"offset": 3490742272,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.9.attn_v.weight",
"offset": 3502276608,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.20.attn_norm.weight",
"offset": 3505160192,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 3505176576,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.20.ffn_up.weight",
"offset": 3545546752,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 3585916928,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.21.attn_norm.weight",
"offset": 3585933312,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 3585949696,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 3634118656,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.21.ffn_up.weight",
"offset": 3674488832,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 3714859008,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 3714875392,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.21.attn_output.weight",
"offset": 3717758976,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.21.attn_q.weight",
"offset": 3729293312,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.21.attn_v.weight",
"offset": 3740827648,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 3744268288,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 3744284672,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 3784654848,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.22.ffn_up.weight",
"offset": 3825025024,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 3865395200,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 3865411584,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.22.attn_output.weight",
"offset": 3868295168,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.22.attn_q.weight",
"offset": 3879829504,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.22.attn_v.weight",
"offset": 3891363840,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.23.attn_norm.weight",
"offset": 3894247424,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 3894263808,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 3934633984,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.23.ffn_up.weight",
"offset": 3975004160,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 4015374336,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 4015390720,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.23.attn_output.weight",
"offset": 4018274304,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.23.attn_q.weight",
"offset": 4029808640,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.23.attn_v.weight",
"offset": 4041342976,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.24.attn_norm.weight",
"offset": 4044226560,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 4044242944,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 4092411904,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.24.ffn_up.weight",
"offset": 4132782080,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 4173152256,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 4173168640,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.24.attn_output.weight",
"offset": 4176052224,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.24.attn_q.weight",
"offset": 4187586560,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.24.attn_v.weight",
"offset": 4199120896,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 4202561536,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 4202577920,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 4242948096,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.25.ffn_up.weight",
"offset": 4283318272,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 4323688448,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 4323704832,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.25.attn_output.weight",
"offset": 4326588416,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.25.attn_q.weight",
"offset": 4338122752,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.25.attn_v.weight",
"offset": 4349657088,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.26.attn_norm.weight",
"offset": 4352540672,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 4352557056,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 4392927232,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.26.ffn_up.weight",
"offset": 4433297408,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 4473667584,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 4473683968,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.26.attn_output.weight",
"offset": 4476567552,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.26.attn_q.weight",
"offset": 4488101888,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.26.attn_v.weight",
"offset": 4499636224,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.27.attn_norm.weight",
"offset": 4502519808,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 4502536192,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 4550705152,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.27.ffn_up.weight",
"offset": 4591075328,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 4631445504,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 4631461888,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.27.attn_output.weight",
"offset": 4634345472,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.27.attn_q.weight",
"offset": 4645879808,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.27.attn_v.weight",
"offset": 4657414144,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 4660854784,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 4660871168,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 4709040128,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.28.ffn_up.weight",
"offset": 4749410304,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 4789780480,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 4789796864,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.28.attn_output.weight",
"offset": 4792680448,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.28.attn_q.weight",
"offset": 4804214784,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.28.attn_v.weight",
"offset": 4815749120,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 4819189760,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 4819206144,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 4867375104,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.29.ffn_up.weight",
"offset": 4907745280,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 4948115456,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 4948131840,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.29.attn_output.weight",
"offset": 4951015424,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.29.attn_q.weight",
"offset": 4962549760,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.29.attn_v.weight",
"offset": 4974084096,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 4977524736,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 4977541120,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 5025710080,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.30.ffn_up.weight",
"offset": 5066080256,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 5106450432,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 5106466816,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.30.attn_output.weight",
"offset": 5109350400,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.30.attn_q.weight",
"offset": 5120884736,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.30.attn_v.weight",
"offset": 5132419072,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 5135859712,
"shape": [
4096,
14336
],
"size": 3440640,
"type": 13
},
{
"name": "blk.31.ffn_up.weight",
"offset": 5176229888,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.31.attn_k.weight",
"offset": 5216600064,
"shape": [
4096,
1024
],
"size": 40370176,
"type": 13
},
{
"name": "blk.31.attn_output.weight",
"offset": 5219483648,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.31.attn_q.weight",
"offset": 5231017984,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.31.attn_v.weight",
"offset": 5242552320,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "\u0000\u0000`_R\u0001\u0000\u0000\u0000\u0016\u0000\u0000\u0000",
"offset": 5245992960,
"shape": [
4096,
128256
],
"size": 3440640,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 5676933120,
"shape": [
4096
],
"size": 430940160,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 5676949504,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 5725118464,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 5725134848,
"shape": [
4096
],
"size": 16384,
"type": 0
}
],
"version": 3
}