Hermes-2 Θ is a merged and then further RLHF'ed version our excellent Hermes 2 Pro model and Meta's Llama-3 Instruct model to form a new model, Hermes-2 Θ, combining the best of both worlds of each model.
8B
312 Pulls Updated 6 weeks ago
3605f0d34778 · 16GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 1,
"general.name": "Hermes-2-Pro-Llama-3-Instruct-Merged-DPO",
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128003,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.pre": "llama-bpe",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 1050673152,
"shape": [
4096
],
"size": 1050673152,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 1050689536,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 1168130048,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.0.ffn_up.weight",
"offset": 1285570560,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 1403011072,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 1403027456,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.0.attn_output.weight",
"offset": 1411416064,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.0.attn_q.weight",
"offset": 1444970496,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.0.attn_v.weight",
"offset": 1478524928,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.1.attn_norm.weight",
"offset": 1486913536,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 1486929920,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 1604370432,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.1.ffn_up.weight",
"offset": 1721810944,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 1839251456,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 1839267840,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.1.attn_output.weight",
"offset": 1847656448,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.1.attn_q.weight",
"offset": 1881210880,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.1.attn_v.weight",
"offset": 1914765312,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.2.attn_norm.weight",
"offset": 1923153920,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 1923170304,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 2040610816,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.2.ffn_up.weight",
"offset": 2158051328,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 2275491840,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 2275508224,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.2.attn_output.weight",
"offset": 2283896832,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.2.attn_q.weight",
"offset": 2317451264,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.2.attn_v.weight",
"offset": 2351005696,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.3.attn_norm.weight",
"offset": 2359394304,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 2359410688,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 2476851200,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.3.ffn_up.weight",
"offset": 2594291712,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 2711732224,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 2711748608,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.3.attn_output.weight",
"offset": 2720137216,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.3.attn_q.weight",
"offset": 2753691648,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.3.attn_v.weight",
"offset": 2787246080,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.4.attn_norm.weight",
"offset": 2795634688,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 2795651072,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 2913091584,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.4.ffn_up.weight",
"offset": 3030532096,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 3147972608,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 3147988992,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.4.attn_output.weight",
"offset": 3156377600,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.4.attn_q.weight",
"offset": 3189932032,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.4.attn_v.weight",
"offset": 3223486464,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.5.attn_norm.weight",
"offset": 3231875072,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 3231891456,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 3349331968,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.5.ffn_up.weight",
"offset": 3466772480,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 3584212992,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 3584229376,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.5.attn_output.weight",
"offset": 3592617984,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.5.attn_q.weight",
"offset": 3626172416,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.5.attn_v.weight",
"offset": 3659726848,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.6.attn_norm.weight",
"offset": 3668115456,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 3668131840,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 3785572352,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.6.ffn_up.weight",
"offset": 3903012864,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 4020453376,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 4020469760,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.6.attn_output.weight",
"offset": 4028858368,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.6.attn_q.weight",
"offset": 4062412800,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.6.attn_v.weight",
"offset": 4095967232,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.7.attn_norm.weight",
"offset": 4104355840,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 4104372224,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 4221812736,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.7.ffn_up.weight",
"offset": 4339253248,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 4456693760,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 4456710144,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.7.attn_output.weight",
"offset": 4465098752,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.7.attn_q.weight",
"offset": 4498653184,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.7.attn_v.weight",
"offset": 4532207616,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.8.attn_norm.weight",
"offset": 4540596224,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 4540612608,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 4658053120,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.8.ffn_up.weight",
"offset": 4775493632,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 4892934144,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 4892950528,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.8.attn_output.weight",
"offset": 4901339136,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.8.attn_q.weight",
"offset": 4934893568,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.8.attn_v.weight",
"offset": 4968448000,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.10.attn_norm.weight",
"offset": 4976836608,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 4976852992,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 5094293504,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.10.ffn_up.weight",
"offset": 5211734016,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 5329174528,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 5329190912,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.10.attn_output.weight",
"offset": 5337579520,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.10.attn_q.weight",
"offset": 5371133952,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.10.attn_v.weight",
"offset": 5404688384,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.11.attn_norm.weight",
"offset": 5413076992,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 5413093376,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 5530533888,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.11.ffn_up.weight",
"offset": 5647974400,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 5765414912,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 5765431296,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.11.attn_output.weight",
"offset": 5773819904,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.11.attn_q.weight",
"offset": 5807374336,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.11.attn_v.weight",
"offset": 5840928768,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.12.attn_norm.weight",
"offset": 5849317376,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 5849333760,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 5966774272,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.12.ffn_up.weight",
"offset": 6084214784,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 6201655296,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 6201671680,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.12.attn_output.weight",
"offset": 6210060288,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.12.attn_q.weight",
"offset": 6243614720,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.12.attn_v.weight",
"offset": 6277169152,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.13.attn_norm.weight",
"offset": 6285557760,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 6285574144,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 6403014656,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.13.ffn_up.weight",
"offset": 6520455168,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 6637895680,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 6637912064,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.13.attn_output.weight",
"offset": 6646300672,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.13.attn_q.weight",
"offset": 6679855104,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.13.attn_v.weight",
"offset": 6713409536,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.14.attn_norm.weight",
"offset": 6721798144,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 6721814528,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 6839255040,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.14.ffn_up.weight",
"offset": 6956695552,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 7074136064,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 7074152448,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.14.attn_output.weight",
"offset": 7082541056,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.14.attn_q.weight",
"offset": 7116095488,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.14.attn_v.weight",
"offset": 7149649920,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.15.attn_norm.weight",
"offset": 7158038528,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 7158054912,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 7275495424,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.15.ffn_up.weight",
"offset": 7392935936,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 7510376448,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 7510392832,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.15.attn_output.weight",
"offset": 7518781440,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.15.attn_q.weight",
"offset": 7552335872,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.15.attn_v.weight",
"offset": 7585890304,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.16.attn_norm.weight",
"offset": 7594278912,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 7594295296,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 7711735808,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.16.ffn_up.weight",
"offset": 7829176320,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 7946616832,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 7946633216,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.16.attn_output.weight",
"offset": 7955021824,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.16.attn_q.weight",
"offset": 7988576256,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.16.attn_v.weight",
"offset": 8022130688,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.17.attn_norm.weight",
"offset": 8030519296,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 8030535680,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 8147976192,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.17.ffn_up.weight",
"offset": 8265416704,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 8382857216,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 8382873600,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.17.attn_output.weight",
"offset": 8391262208,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.17.attn_q.weight",
"offset": 8424816640,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.17.attn_v.weight",
"offset": 8458371072,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.18.attn_norm.weight",
"offset": 8466759680,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 8466776064,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 8584216576,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.18.ffn_up.weight",
"offset": 8701657088,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 8819097600,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 8819113984,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.18.attn_output.weight",
"offset": 8827502592,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.18.attn_q.weight",
"offset": 8861057024,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.18.attn_v.weight",
"offset": 8894611456,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.19.attn_norm.weight",
"offset": 8903000064,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 8903016448,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 9020456960,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.19.ffn_up.weight",
"offset": 9137897472,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 9255337984,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 9255354368,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.19.attn_output.weight",
"offset": 9263742976,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.19.attn_q.weight",
"offset": 9297297408,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.19.attn_v.weight",
"offset": 9330851840,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 9339240448,
"shape": [
4096,
14336
],
"size": 8388608,
"type": 1
},
{
"name": "blk.20.attn_k.weight",
"offset": 9456680960,
"shape": [
4096,
1024
],
"size": 117440512,
"type": 1
},
{
"name": "blk.20.attn_output.weight",
"offset": 9465069568,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.20.attn_q.weight",
"offset": 9498624000,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.20.attn_v.weight",
"offset": 9532178432,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.9.attn_norm.weight",
"offset": 9540567040,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 9540583424,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 9658023936,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.9.ffn_up.weight",
"offset": 9775464448,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 9892904960,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 9892921344,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.9.attn_output.weight",
"offset": 9901309952,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.9.attn_q.weight",
"offset": 9934864384,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.9.attn_v.weight",
"offset": 9968418816,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.20.attn_norm.weight",
"offset": 9976807424,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 9976823808,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.20.ffn_up.weight",
"offset": 10094264320,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 10211704832,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.21.attn_norm.weight",
"offset": 10211721216,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 10211737600,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 10329178112,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.21.ffn_up.weight",
"offset": 10446618624,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 10564059136,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 10564075520,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.21.attn_output.weight",
"offset": 10572464128,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.21.attn_q.weight",
"offset": 10606018560,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.21.attn_v.weight",
"offset": 10639572992,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.22.attn_norm.weight",
"offset": 10647961600,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 10647977984,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 10765418496,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.22.ffn_up.weight",
"offset": 10882859008,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 11000299520,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 11000315904,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.22.attn_output.weight",
"offset": 11008704512,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.22.attn_q.weight",
"offset": 11042258944,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.22.attn_v.weight",
"offset": 11075813376,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.23.attn_norm.weight",
"offset": 11084201984,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 11084218368,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 11201658880,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.23.ffn_up.weight",
"offset": 11319099392,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 11436539904,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 11436556288,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.23.attn_output.weight",
"offset": 11444944896,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.23.attn_q.weight",
"offset": 11478499328,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.23.attn_v.weight",
"offset": 11512053760,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.24.attn_norm.weight",
"offset": 11520442368,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 11520458752,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 11637899264,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.24.ffn_up.weight",
"offset": 11755339776,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 11872780288,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 11872796672,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.24.attn_output.weight",
"offset": 11881185280,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.24.attn_q.weight",
"offset": 11914739712,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.24.attn_v.weight",
"offset": 11948294144,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.25.attn_norm.weight",
"offset": 11956682752,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 11956699136,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 12074139648,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.25.ffn_up.weight",
"offset": 12191580160,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 12309020672,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 12309037056,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.25.attn_output.weight",
"offset": 12317425664,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.25.attn_q.weight",
"offset": 12350980096,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.25.attn_v.weight",
"offset": 12384534528,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.26.attn_norm.weight",
"offset": 12392923136,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 12392939520,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 12510380032,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.26.ffn_up.weight",
"offset": 12627820544,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 12745261056,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 12745277440,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.26.attn_output.weight",
"offset": 12753666048,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.26.attn_q.weight",
"offset": 12787220480,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.26.attn_v.weight",
"offset": 12820774912,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.27.attn_norm.weight",
"offset": 12829163520,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 12829179904,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 12946620416,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.27.ffn_up.weight",
"offset": 13064060928,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 13181501440,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 13181517824,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.27.attn_output.weight",
"offset": 13189906432,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.27.attn_q.weight",
"offset": 13223460864,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.27.attn_v.weight",
"offset": 13257015296,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.28.attn_norm.weight",
"offset": 13265403904,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 13265420288,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 13382860800,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.28.ffn_up.weight",
"offset": 13500301312,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 13617741824,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 13617758208,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.28.attn_output.weight",
"offset": 13626146816,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.28.attn_q.weight",
"offset": 13659701248,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.28.attn_v.weight",
"offset": 13693255680,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.29.attn_norm.weight",
"offset": 13701644288,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 13701660672,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 13819101184,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.29.ffn_up.weight",
"offset": 13936541696,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 14053982208,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 14053998592,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.29.attn_output.weight",
"offset": 14062387200,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.29.attn_q.weight",
"offset": 14095941632,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.29.attn_v.weight",
"offset": 14129496064,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.30.attn_norm.weight",
"offset": 14137884672,
"shape": [
4096
],
"size": 8388608,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 14137901056,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 14255341568,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.30.ffn_up.weight",
"offset": 14372782080,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 14490222592,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 14490238976,
"shape": [
4096,
1024
],
"size": 16384,
"type": 1
},
{
"name": "blk.30.attn_output.weight",
"offset": 14498627584,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.30.attn_q.weight",
"offset": 14532182016,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.30.attn_v.weight",
"offset": 14565736448,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 14574125056,
"shape": [
4096,
14336
],
"size": 8388608,
"type": 1
},
{
"name": "blk.31.ffn_up.weight",
"offset": 14691565568,
"shape": [
4096,
14336
],
"size": 117440512,
"type": 1
},
{
"name": "blk.31.attn_k.weight",
"offset": 14809006080,
"shape": [
4096,
1024
],
"size": 117440512,
"type": 1
},
{
"name": "blk.31.attn_output.weight",
"offset": 14817394688,
"shape": [
4096,
4096
],
"size": 8388608,
"type": 1
},
{
"name": "blk.31.attn_q.weight",
"offset": 14850949120,
"shape": [
4096,
4096
],
"size": 33554432,
"type": 1
},
{
"name": "blk.31.attn_v.weight",
"offset": 14884503552,
"shape": [
4096,
1024
],
"size": 33554432,
"type": 1
},
{
"name": "output.weight",
"offset": 14892892160,
"shape": [
4096,
128256
],
"size": 8388608,
"type": 1
},
{
"name": "blk.31.attn_norm.weight",
"offset": 15943565312,
"shape": [
4096
],
"size": 1050673152,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 15943581696,
"shape": [
14336,
4096
],
"size": 16384,
"type": 1
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 16061022208,
"shape": [
4096
],
"size": 117440512,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 16061038592,
"shape": [
4096
],
"size": 16384,
"type": 0
}
],
"version": 3
}