Lexi is uncensored, which makes the model compliant. You are advised to implement your own alignment layer before exposing the model as a service. It will be highly compliant with any requests, even unethical ones.
8B
2,701 Pulls Updated 5 weeks ago
cfce48ca57f0 · 16GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 1,
"general.name": "bk",
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128009,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 1050673152,
"shape": [
4096
],
"size": 1050673152,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 1050689536,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 1168130048,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.0.ffn_up.weight",
"offset": 1285570560,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 1403011072,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 1403027456,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.0.attn_output.weight",
"offset": 1411416064,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.0.attn_q.weight",
"offset": 1444970496,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.0.attn_v.weight",
"offset": 1478524928,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.1.attn_norm.weight",
"offset": 1486913536,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 1486929920,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 1604370432,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.1.ffn_up.weight",
"offset": 1721810944,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 1839251456,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 1839267840,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.1.attn_output.weight",
"offset": 1847656448,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.1.attn_q.weight",
"offset": 1881210880,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.1.attn_v.weight",
"offset": 1914765312,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.2.attn_norm.weight",
"offset": 1923153920,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 1923170304,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 2040610816,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.2.ffn_up.weight",
"offset": 2158051328,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 2275491840,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 2275508224,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.2.attn_output.weight",
"offset": 2283896832,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.2.attn_q.weight",
"offset": 2317451264,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.2.attn_v.weight",
"offset": 2351005696,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.3.attn_norm.weight",
"offset": 2359394304,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 2359410688,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 2476851200,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.3.ffn_up.weight",
"offset": 2594291712,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 2711732224,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 2711748608,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.3.attn_output.weight",
"offset": 2720137216,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.3.attn_q.weight",
"offset": 2753691648,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.3.attn_v.weight",
"offset": 2787246080,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.4.attn_norm.weight",
"offset": 2795634688,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 2795651072,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 2913091584,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.4.ffn_up.weight",
"offset": 3030532096,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 3147972608,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 3147988992,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.4.attn_output.weight",
"offset": 3156377600,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.4.attn_q.weight",
"offset": 3189932032,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.4.attn_v.weight",
"offset": 3223486464,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.5.attn_norm.weight",
"offset": 3231875072,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 3231891456,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 3349331968,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.5.ffn_up.weight",
"offset": 3466772480,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 3584212992,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 3584229376,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.5.attn_output.weight",
"offset": 3592617984,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.5.attn_q.weight",
"offset": 3626172416,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.5.attn_v.weight",
"offset": 3659726848,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.6.attn_norm.weight",
"offset": 3668115456,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 3668131840,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 3785572352,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.6.ffn_up.weight",
"offset": 3903012864,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 4020453376,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 4020469760,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.6.attn_output.weight",
"offset": 4028858368,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.6.attn_q.weight",
"offset": 4062412800,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.6.attn_v.weight",
"offset": 4095967232,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.7.attn_norm.weight",
"offset": 4104355840,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 4104372224,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 4221812736,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.7.ffn_up.weight",
"offset": 4339253248,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 4456693760,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 4456710144,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.7.attn_output.weight",
"offset": 4465098752,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.7.attn_q.weight",
"offset": 4498653184,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.7.attn_v.weight",
"offset": 4532207616,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.10.attn_norm.weight",
"offset": 4540596224,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 4540612608,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 4658053120,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.10.ffn_up.weight",
"offset": 4775493632,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 4892934144,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 4892950528,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.10.attn_output.weight",
"offset": 4901339136,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.10.attn_q.weight",
"offset": 4934893568,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "weight\u0002\u0000\u0000\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\u00008",
"offset": 4968448000,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.11.attn_norm.weight",
"offset": 4976836608,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 4976852992,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 5094293504,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.11.ffn_up.weight",
"offset": 5211734016,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 5329174528,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 5329190912,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.11.attn_output.weight",
"offset": 5337579520,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.11.attn_q.weight",
"offset": 5371133952,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.11.attn_v.weight",
"offset": 5404688384,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.12.attn_norm.weight",
"offset": 5413076992,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 5413093376,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 5530533888,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.12.ffn_up.weight",
"offset": 5647974400,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 5765414912,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 5765431296,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.12.attn_output.weight",
"offset": 5773819904,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.12.attn_q.weight",
"offset": 5807374336,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.12.attn_v.weight",
"offset": 5840928768,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.13.attn_norm.weight",
"offset": 5849317376,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 5849333760,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 5966774272,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.13.ffn_up.weight",
"offset": 6084214784,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 6201655296,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 6201671680,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.13.attn_output.weight",
"offset": 6210060288,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.13.attn_q.weight",
"offset": 6243614720,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.13.attn_v.weight",
"offset": 6277169152,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.14.attn_norm.weight",
"offset": 6285557760,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 6285574144,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 6403014656,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.14.ffn_up.weight",
"offset": 6520455168,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 6637895680,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 6637912064,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.14.attn_output.weight",
"offset": 6646300672,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.14.attn_q.weight",
"offset": 6679855104,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.14.attn_v.weight",
"offset": 6713409536,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.15.attn_norm.weight",
"offset": 6721798144,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 6721814528,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 6839255040,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.15.ffn_up.weight",
"offset": 6956695552,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 7074136064,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 7074152448,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.15.attn_output.weight",
"offset": 7082541056,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.15.attn_q.weight",
"offset": 7116095488,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.15.attn_v.weight",
"offset": 7149649920,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.16.attn_norm.weight",
"offset": 7158038528,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 7158054912,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 7275495424,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.16.ffn_up.weight",
"offset": 7392935936,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 7510376448,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 7510392832,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.16.attn_output.weight",
"offset": 7518781440,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.16.attn_q.weight",
"offset": 7552335872,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.16.attn_v.weight",
"offset": 7585890304,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.17.attn_norm.weight",
"offset": 7594278912,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 7594295296,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 7711735808,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.17.ffn_up.weight",
"offset": 7829176320,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 7946616832,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 7946633216,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.17.attn_output.weight",
"offset": 7955021824,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.17.attn_q.weight",
"offset": 7988576256,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.17.attn_v.weight",
"offset": 8022130688,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.18.attn_k.weight",
"offset": 8030519296,
"shape": [
4096,
1024
],
"size": 8388608,
"type": 1
},
{
"name": "blk.18.attn_output.weight",
"offset": 8038907904,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.18.attn_q.weight",
"offset": 8072462336,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.18.attn_v.weight",
"offset": 8106016768,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.8.attn_norm.weight",
"offset": 8114405376,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 8114421760,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 8231862272,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.8.ffn_up.weight",
"offset": 8349302784,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 8466743296,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 8466759680,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.8.attn_output.weight",
"offset": 8475148288,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.8.attn_q.weight",
"offset": 8508702720,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.8.attn_v.weight",
"offset": 8542257152,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.9.attn_norm.weight",
"offset": 8550645760,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 8550662144,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 8668102656,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.9.ffn_up.weight",
"offset": 8785543168,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 8902983680,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 8903000064,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.9.attn_output.weight",
"offset": 8911388672,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.9.attn_q.weight",
"offset": 8944943104,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.9.attn_v.weight",
"offset": 8978497536,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.18.attn_norm.weight",
"offset": 8986886144,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 8986902528,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 9104343040,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.18.ffn_up.weight",
"offset": 9221783552,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 9339224064,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.19.attn_norm.weight",
"offset": 9339240448,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 9339256832,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 9456697344,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.19.ffn_up.weight",
"offset": 9574137856,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 9691578368,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 9691594752,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.19.attn_output.weight",
"offset": 9699983360,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.19.attn_q.weight",
"offset": 9733537792,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.19.attn_v.weight",
"offset": 9767092224,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.20.attn_norm.weight",
"offset": 9775480832,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 9775497216,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 9892937728,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.20.ffn_up.weight",
"offset": 10010378240,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 10127818752,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 10127835136,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.20.attn_output.weight",
"offset": 10136223744,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.20.attn_q.weight",
"offset": 10169778176,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.20.attn_v.weight",
"offset": 10203332608,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.21.attn_norm.weight",
"offset": 10211721216,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 10211737600,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 10329178112,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.21.ffn_up.weight",
"offset": 10446618624,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 10564059136,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 10564075520,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.21.attn_output.weight",
"offset": 10572464128,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.21.attn_q.weight",
"offset": 10606018560,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.21.attn_v.weight",
"offset": 10639572992,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.22.attn_norm.weight",
"offset": 10647961600,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 10647977984,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 10765418496,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.22.ffn_up.weight",
"offset": 10882859008,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 11000299520,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 11000315904,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.22.attn_output.weight",
"offset": 11008704512,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.22.attn_q.weight",
"offset": 11042258944,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.22.attn_v.weight",
"offset": 11075813376,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.23.attn_norm.weight",
"offset": 11084201984,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 11084218368,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 11201658880,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.23.ffn_up.weight",
"offset": 11319099392,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 11436539904,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 11436556288,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.23.attn_output.weight",
"offset": 11444944896,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.23.attn_q.weight",
"offset": 11478499328,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.23.attn_v.weight",
"offset": 11512053760,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.24.attn_norm.weight",
"offset": 11520442368,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "\ufffd\ufffd\ufffd\u001a\u0010\ufffd\ufffd\ufffd\u0000\ufffdh\u001eP\u00178\u0015\ufffd\ufffd \u0016\u0000\u001a",
"offset": 11520458752,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 11637899264,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.24.ffn_up.weight",
"offset": 11755339776,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 11872780288,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 11872796672,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.24.attn_output.weight",
"offset": 11881185280,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.24.attn_q.weight",
"offset": 11914739712,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.24.attn_v.weight",
"offset": 11948294144,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.25.attn_norm.weight",
"offset": 11956682752,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 11956699136,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 12074139648,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.25.ffn_up.weight",
"offset": 12191580160,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 12309020672,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 12309037056,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.25.attn_output.weight",
"offset": 12317425664,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.25.attn_q.weight",
"offset": 12350980096,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.25.attn_v.weight",
"offset": 12384534528,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.26.attn_norm.weight",
"offset": 12392923136,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 12392939520,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 12510380032,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.26.ffn_up.weight",
"offset": 12627820544,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 12745261056,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 12745277440,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.26.attn_output.weight",
"offset": 12753666048,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.26.attn_q.weight",
"offset": 12787220480,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.26.attn_v.weight",
"offset": 12820774912,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.27.attn_norm.weight",
"offset": 12829163520,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 12829179904,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 12946620416,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.27.ffn_up.weight",
"offset": 13064060928,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 13181501440,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 13181517824,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.27.attn_output.weight",
"offset": 13189906432,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.27.attn_q.weight",
"offset": 13223460864,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.27.attn_v.weight",
"offset": 13257015296,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 13265403904,
"shape": [
4096,
14336
],
"size": 8388608,
"type": 1
},
{
"name": "blk.28.attn_k.weight",
"offset": 13382844416,
"shape": [
4096,
1024
],
"size": 117440512,
"type": 1
},
{
"name": "blk.28.attn_output.weight",
"offset": 13391233024,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.28.attn_q.weight",
"offset": 13424787456,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.28.attn_v.weight",
"offset": 13458341888,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "output.weight",
"offset": 13466730496,
"shape": [
4096,
128256
],
"size": 8388608,
"type": 1
},
{
"name": "blk.28.attn_norm.weight",
"offset": 14517403648,
"shape": [
4096
],
"size": 1050673152,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 14517420032,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.28.ffn_up.weight",
"offset": 14634860544,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 14752301056,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.29.attn_norm.weight",
"offset": 14752317440,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 14752333824,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 14869774336,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.29.ffn_up.weight",
"offset": 14987214848,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 15104655360,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 15104671744,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.29.attn_output.weight",
"offset": 15113060352,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.29.attn_q.weight",
"offset": 15146614784,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.29.attn_v.weight",
"offset": 15180169216,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.30.attn_norm.weight",
"offset": 15188557824,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 15188574208,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 15306014720,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.30.ffn_up.weight",
"offset": 15423455232,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 15540895744,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 15540912128,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.30.attn_output.weight",
"offset": 15549300736,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.30.attn_q.weight",
"offset": 15582855168,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.30.attn_v.weight",
"offset": 15616409600,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.31.attn_norm.weight",
"offset": 15624798208,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 15624814592,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 15742255104,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.31.ffn_up.weight",
"offset": 15859695616,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 15977136128,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 15977152512,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.31.attn_output.weight",
"offset": 15985541120,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.31.attn_q.weight",
"offset": 16019095552,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.31.attn_v.weight",
"offset": 16052649984,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "output_norm.weight",
"offset": 16061038592,
"shape": [
4096
],
"size": 8388608,
"type": 0
}
],
"version": 3
}