Lexi is uncensored, which makes the model compliant. You are advised to implement your own alignment layer before exposing the model as a service. It will be highly compliant with any requests, even unethical ones.
8B
2,804 Pulls Updated 5 weeks ago
fc47b8a415a5 · 6.6GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 18,
"general.name": "LexiFun",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 430940160,
"shape": [
4096
],
"size": 430940160,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 430956544,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 479125504,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.0.ffn_up.weight",
"offset": 527294464,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 575463424,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 575479808,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.0.attn_output.weight",
"offset": 578920448,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.0.attn_q.weight",
"offset": 592683008,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.0.attn_v.weight",
"offset": 606445568,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 609886208,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 609902592,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 658071552,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.1.ffn_up.weight",
"offset": 706240512,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 754409472,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 754425856,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.attn_output.weight",
"offset": 757866496,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.1.attn_q.weight",
"offset": 771629056,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.1.attn_v.weight",
"offset": 785391616,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 788832256,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 788848640,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 837017600,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.2.ffn_up.weight",
"offset": 885186560,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 933355520,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 933371904,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.attn_output.weight",
"offset": 936812544,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.2.attn_q.weight",
"offset": 950575104,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.2.attn_v.weight",
"offset": 964337664,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 967778304,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 967794688,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 1015963648,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.3.ffn_up.weight",
"offset": 1064132608,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 1112301568,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 1112317952,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.3.attn_output.weight",
"offset": 1115758592,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.3.attn_q.weight",
"offset": 1129521152,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.3.attn_v.weight",
"offset": 1143283712,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 1146724352,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 1146740736,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 1194909696,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1243078656,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1291247616,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 1291264000,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.4.attn_output.weight",
"offset": 1294704640,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.4.attn_q.weight",
"offset": 1308467200,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.4.attn_v.weight",
"offset": 1322229760,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.5.attn_norm.weight",
"offset": 1325670400,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 1325686784,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1373855744,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1422024704,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1470193664,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1470210048,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.5.attn_output.weight",
"offset": 1473650688,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.5.attn_q.weight",
"offset": 1487413248,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.5.attn_v.weight",
"offset": 1501175808,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1504616448,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1504632832,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 1552801792,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.6.ffn_up.weight",
"offset": 1600970752,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 1649139712,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 1649156096,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.attn_output.weight",
"offset": 1652596736,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.6.attn_q.weight",
"offset": 1666359296,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.6.attn_v.weight",
"offset": 1680121856,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 1683562496,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 1683578880,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 1731747840,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.7.ffn_up.weight",
"offset": 1779916800,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 1828085760,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 1828102144,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.7.attn_output.weight",
"offset": 1831542784,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.7.attn_q.weight",
"offset": 1845305344,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.7.attn_v.weight",
"offset": 1859067904,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 1862508544,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 1862524928,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 1910693888,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.8.ffn_up.weight",
"offset": 1958862848,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 2007031808,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 2007048192,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.8.attn_output.weight",
"offset": 2010488832,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.8.attn_q.weight",
"offset": 2024251392,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.8.attn_v.weight",
"offset": 2038013952,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.10.attn_norm.weight",
"offset": 2041454592,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 2041470976,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 2089639936,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.10.ffn_up.weight",
"offset": 2137808896,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 2185977856,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 2185994240,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.10.attn_output.weight",
"offset": 2189434880,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.10.attn_q.weight",
"offset": 2203197440,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.10.attn_v.weight",
"offset": 2216960000,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 2220400640,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 2220417024,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 2268585984,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.11.ffn_up.weight",
"offset": 2316754944,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 2364923904,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 2364940288,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.11.attn_output.weight",
"offset": 2368380928,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.11.attn_q.weight",
"offset": 2382143488,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.11.attn_v.weight",
"offset": 2395906048,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 2399346688,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 2399363072,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 2447532032,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.12.ffn_up.weight",
"offset": 2495700992,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 2543869952,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 2543886336,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.12.attn_output.weight",
"offset": 2547326976,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.12.attn_q.weight",
"offset": 2561089536,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.12.attn_v.weight",
"offset": 2574852096,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.13.attn_norm.weight",
"offset": 2578292736,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 2578309120,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 2626478080,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.13.ffn_up.weight",
"offset": 2674647040,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 2722816000,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 2722832384,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.13.attn_output.weight",
"offset": 2726273024,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.13.attn_q.weight",
"offset": 2740035584,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.13.attn_v.weight",
"offset": 2753798144,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 2757238784,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 2757255168,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 2805424128,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.14.ffn_up.weight",
"offset": 2853593088,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 2901762048,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 2901778432,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.14.attn_output.weight",
"offset": 2905219072,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.14.attn_q.weight",
"offset": 2918981632,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.14.attn_v.weight",
"offset": 2932744192,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 2936184832,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 2936201216,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 2984370176,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.15.ffn_up.weight",
"offset": 3032539136,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 3080708096,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 3080724480,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.15.attn_output.weight",
"offset": 3084165120,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.15.attn_q.weight",
"offset": 3097927680,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.15.attn_v.weight",
"offset": 3111690240,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.16.attn_norm.weight",
"offset": 3115130880,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 3115147264,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 3163316224,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.16.ffn_up.weight",
"offset": 3211485184,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 3259654144,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 3259670528,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.16.attn_output.weight",
"offset": 3263111168,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.16.attn_q.weight",
"offset": 3276873728,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.16.attn_v.weight",
"offset": 3290636288,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 3294076928,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 3294093312,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 3342262272,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.17.ffn_up.weight",
"offset": 3390431232,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 3438600192,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 3438616576,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.17.attn_output.weight",
"offset": 3442057216,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.17.attn_q.weight",
"offset": 3455819776,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.17.attn_v.weight",
"offset": 3469582336,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 3473022976,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 3473039360,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 3521208320,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.18.ffn_up.weight",
"offset": 3569377280,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 3617546240,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 3617562624,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.18.attn_output.weight",
"offset": 3621003264,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.18.attn_q.weight",
"offset": 3634765824,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.18.attn_v.weight",
"offset": 3648528384,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.19.attn_norm.weight",
"offset": 3651969024,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 3651985408,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 3700154368,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.19.ffn_up.weight",
"offset": 3748323328,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 3796492288,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 3796508672,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.19.attn_output.weight",
"offset": 3799949312,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.19.attn_q.weight",
"offset": 3813711872,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.19.attn_v.weight",
"offset": 3827474432,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 3830915072,
"shape": [
4096,
14336
],
"size": 3440640,
"type": 14
},
{
"name": "blk.20.attn_k.weight",
"offset": 3879084032,
"shape": [
4096,
1024
],
"size": 48168960,
"type": 14
},
{
"name": "blk.20.attn_output.weight",
"offset": 3882524672,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.20.attn_q.weight",
"offset": 3896287232,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.20.attn_v.weight",
"offset": 3910049792,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 3913490432,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 3913506816,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 3961675776,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.9.ffn_up.weight",
"offset": 4009844736,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 4058013696,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 4058030080,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.9.attn_output.weight",
"offset": 4061470720,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.9.attn_q.weight",
"offset": 4075233280,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.9.attn_v.weight",
"offset": 4088995840,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 4092436480,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 4092452864,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.20.ffn_up.weight",
"offset": 4140621824,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 4188790784,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.21.attn_norm.weight",
"offset": 4188807168,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 4188823552,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 4236992512,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.21.ffn_up.weight",
"offset": 4285161472,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 4333330432,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 4333346816,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.21.attn_output.weight",
"offset": 4336787456,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.21.attn_q.weight",
"offset": 4350550016,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.21.attn_v.weight",
"offset": 4364312576,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 4367753216,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 4367769600,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 4415938560,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.22.ffn_up.weight",
"offset": 4464107520,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 4512276480,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 4512292864,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.22.attn_output.weight",
"offset": 4515733504,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.22.attn_q.weight",
"offset": 4529496064,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.22.attn_v.weight",
"offset": 4543258624,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 4546699264,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 4546715648,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 4594884608,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.23.ffn_up.weight",
"offset": 4643053568,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 4691222528,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 4691238912,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.23.attn_output.weight",
"offset": 4694679552,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.23.attn_q.weight",
"offset": 4708442112,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.23.attn_v.weight",
"offset": 4722204672,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.24.attn_norm.weight",
"offset": 4725645312,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 4725661696,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 4773830656,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.24.ffn_up.weight",
"offset": 4821999616,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 4870168576,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 4870184960,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.24.attn_output.weight",
"offset": 4873625600,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.24.attn_q.weight",
"offset": 4887388160,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.24.attn_v.weight",
"offset": 4901150720,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 4904591360,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 4904607744,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 4952776704,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.25.ffn_up.weight",
"offset": 5000945664,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 5049114624,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 5049131008,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.25.attn_output.weight",
"offset": 5052571648,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.25.attn_q.weight",
"offset": 5066334208,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.25.attn_v.weight",
"offset": 5080096768,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.26.attn_norm.weight",
"offset": 5083537408,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 5083553792,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 5131722752,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.26.ffn_up.weight",
"offset": 5179891712,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 5228060672,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 5228077056,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.26.attn_output.weight",
"offset": 5231517696,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.26.attn_q.weight",
"offset": 5245280256,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.26.attn_v.weight",
"offset": 5259042816,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.27.attn_norm.weight",
"offset": 5262483456,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 5262499840,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 5310668800,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.27.ffn_up.weight",
"offset": 5358837760,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 5407006720,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 5407023104,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.27.attn_output.weight",
"offset": 5410463744,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.27.attn_q.weight",
"offset": 5424226304,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.27.attn_v.weight",
"offset": 5437988864,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 5441429504,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 5441445888,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 5489614848,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.28.ffn_up.weight",
"offset": 5537783808,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 5585952768,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 5585969152,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.attn_output.weight",
"offset": 5589409792,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.28.attn_q.weight",
"offset": 5603172352,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.28.attn_v.weight",
"offset": 5616934912,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 5620375552,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 5620391936,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 5668560896,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.29.ffn_up.weight",
"offset": 5716729856,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 5764898816,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 5764915200,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.29.attn_output.weight",
"offset": 5768355840,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.29.attn_q.weight",
"offset": 5782118400,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.29.attn_v.weight",
"offset": 5795880960,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 5799321600,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 5799337984,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 5847506944,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.30.ffn_up.weight",
"offset": 5895675904,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 5943844864,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 5943861248,
"shape": [
4096,
1024
],
"size": 16384,
"type": 14
},
{
"name": "blk.30.attn_output.weight",
"offset": 5947301888,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.30.attn_q.weight",
"offset": 5961064448,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.30.attn_v.weight",
"offset": 5974827008,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 5978267648,
"shape": [
4096,
14336
],
"size": 3440640,
"type": 14
},
{
"name": "blk.31.ffn_up.weight",
"offset": 6026436608,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.31.attn_k.weight",
"offset": 6074605568,
"shape": [
4096,
1024
],
"size": 48168960,
"type": 14
},
{
"name": "blk.31.attn_output.weight",
"offset": 6078046208,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.31.attn_q.weight",
"offset": 6091808768,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.31.attn_v.weight",
"offset": 6105571328,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "output.weight",
"offset": 6109011968,
"shape": [
4096,
128256
],
"size": 3440640,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 6539952128,
"shape": [
4096
],
"size": 430940160,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 6539968512,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 6588137472,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 6588153856,
"shape": [
4096
],
"size": 16384,
"type": 0
}
],
"version": 3
}