This new version of Hermes maintains its excellent general task and conversation capabilities - but also excels at Function Calling, JSON Structured Outputs, and has improved on several other metrics as well
8B
177 Pulls Updated 7 weeks ago
107f3f55e26b · 5.7GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 17,
"general.name": "Hermes-2-Pro-Llama-3-8B",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128288,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128003,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.pre": "llama-bpe",
"tokenizer.ggml.token_type": "... (128288 values)",
"tokenizer.ggml.tokens": "... (128288 values)"
},
"num_params": 8030523392,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 361259008,
"shape": [
4096
],
"size": 361259008,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 361275392,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 409444352,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.0.ffn_up.weight",
"offset": 449814528,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 490184704,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 490201088,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.0.attn_output.weight",
"offset": 493084672,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.0.attn_q.weight",
"offset": 504619008,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.0.attn_v.weight",
"offset": 516153344,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 519593984,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 519610368,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 567779328,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.1.ffn_up.weight",
"offset": 608149504,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 648519680,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 648536064,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.1.attn_output.weight",
"offset": 651419648,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.1.attn_q.weight",
"offset": 662953984,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.1.attn_v.weight",
"offset": 674488320,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 677928960,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 677945344,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 726114304,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.2.ffn_up.weight",
"offset": 766484480,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 806854656,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 806871040,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.2.attn_output.weight",
"offset": 809754624,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.2.attn_q.weight",
"offset": 821288960,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.2.attn_v.weight",
"offset": 832823296,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 836263936,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 836280320,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 884449280,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.3.ffn_up.weight",
"offset": 924819456,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 965189632,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 965206016,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.3.attn_output.weight",
"offset": 968089600,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.3.attn_q.weight",
"offset": 979623936,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.3.attn_v.weight",
"offset": 991158272,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 994598912,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 994615296,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 1034985472,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1075355648,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1115725824,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 1115742208,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.4.attn_output.weight",
"offset": 1118625792,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.4.attn_q.weight",
"offset": 1130160128,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.4.attn_v.weight",
"offset": 1141694464,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.5.attn_norm.weight",
"offset": 1144578048,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 1144594432,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1184964608,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1225334784,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1265704960,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1265721344,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.5.attn_output.weight",
"offset": 1268604928,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.5.attn_q.weight",
"offset": 1280139264,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.5.attn_v.weight",
"offset": 1291673600,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1294557184,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1294573568,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 1342742528,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.6.ffn_up.weight",
"offset": 1383112704,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 1423482880,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 1423499264,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.6.attn_output.weight",
"offset": 1426382848,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\r\u0000\u0000",
"offset": 1437917184,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.6.attn_v.weight",
"offset": 1449451520,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 1452892160,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 1452908544,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 1493278720,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.7.ffn_up.weight",
"offset": 1533648896,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 1574019072,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 1574035456,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.7.attn_output.weight",
"offset": 1576919040,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.7.attn_q.weight",
"offset": 1588453376,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.7.attn_v.weight",
"offset": 1599987712,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.8.attn_norm.weight",
"offset": 1602871296,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 1602887680,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 1643257856,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.8.ffn_up.weight",
"offset": 1683628032,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 1723998208,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 1724014592,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.8.attn_output.weight",
"offset": 1726898176,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.8.attn_q.weight",
"offset": 1738432512,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.8.attn_v.weight",
"offset": 1749966848,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.10.attn_norm.weight",
"offset": 1752850432,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 1752866816,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 1801035776,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.10.ffn_up.weight",
"offset": 1841405952,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 1881776128,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 1881792512,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.10.attn_output.weight",
"offset": 1884676096,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.10.attn_q.weight",
"offset": 1896210432,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.10.attn_v.weight",
"offset": 1907744768,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 1911185408,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 1911201792,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 1951571968,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.11.ffn_up.weight",
"offset": 1991942144,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 2032312320,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 2032328704,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.11.attn_output.weight",
"offset": 2035212288,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.11.attn_q.weight",
"offset": 2046746624,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.11.attn_v.weight",
"offset": 2058280960,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.12.attn_norm.weight",
"offset": 2061164544,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 2061180928,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 2101551104,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.12.ffn_up.weight",
"offset": 2141921280,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 2182291456,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 2182307840,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.12.attn_output.weight",
"offset": 2185191424,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.12.attn_q.weight",
"offset": 2196725760,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.12.attn_v.weight",
"offset": 2208260096,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.13.attn_norm.weight",
"offset": 2211143680,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 2211160064,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 2259329024,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.13.ffn_up.weight",
"offset": 2299699200,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 2340069376,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 2340085760,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.13.attn_output.weight",
"offset": 2342969344,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.13.attn_q.weight",
"offset": 2354503680,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.13.attn_v.weight",
"offset": 2366038016,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 2369478656,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 2369495040,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 2409865216,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.14.ffn_up.weight",
"offset": 2450235392,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 2490605568,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 2490621952,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.14.attn_output.weight",
"offset": 2493505536,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.14.attn_q.weight",
"offset": 2505039872,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.14.attn_v.weight",
"offset": 2516574208,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.15.attn_norm.weight",
"offset": 2519457792,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 2519474176,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 2559844352,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.15.ffn_up.weight",
"offset": 2600214528,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 2640584704,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 2640601088,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.15.attn_output.weight",
"offset": 2643484672,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.15.attn_q.weight",
"offset": 2655019008,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.15.attn_v.weight",
"offset": 2666553344,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.16.attn_norm.weight",
"offset": 2669436928,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 2669453312,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 2717622272,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.16.ffn_up.weight",
"offset": 2757992448,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 2798362624,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 2798379008,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.16.attn_output.weight",
"offset": 2801262592,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.16.attn_q.weight",
"offset": 2812796928,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.16.attn_v.weight",
"offset": 2824331264,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 2827771904,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 2827788288,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 2868158464,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.17.ffn_up.weight",
"offset": 2908528640,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 2948898816,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 2948915200,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.17.attn_output.weight",
"offset": 2951798784,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.17.attn_q.weight",
"offset": 2963333120,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.17.attn_v.weight",
"offset": 2974867456,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.18.attn_norm.weight",
"offset": 2977751040,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 2977767424,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 3018137600,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.18.ffn_up.weight",
"offset": 3058507776,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 3098877952,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 3098894336,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.18.attn_output.weight",
"offset": 3101777920,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.18.attn_q.weight",
"offset": 3113312256,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.18.attn_v.weight",
"offset": 3124846592,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.19.attn_norm.weight",
"offset": 3127730176,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 3127746560,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 3175915520,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.19.ffn_up.weight",
"offset": 3216285696,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 3256655872,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 3256672256,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.19.attn_output.weight",
"offset": 3259555840,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.19.attn_q.weight",
"offset": 3271090176,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.19.attn_v.weight",
"offset": 3282624512,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 3286065152,
"shape": [
4096,
14336
],
"size": 3440640,
"type": 13
},
{
"name": "blk.20.attn_k.weight",
"offset": 3326435328,
"shape": [
4096,
1024
],
"size": 40370176,
"type": 13
},
{
"name": "blk.20.attn_output.weight",
"offset": 3329318912,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.20.attn_q.weight",
"offset": 3340853248,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.20.attn_v.weight",
"offset": 3352387584,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.9.attn_norm.weight",
"offset": 3355271168,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 3355287552,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 3395657728,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.9.ffn_up.weight",
"offset": 3436027904,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 3476398080,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 3476414464,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.9.attn_output.weight",
"offset": 3479298048,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.9.attn_q.weight",
"offset": 3490832384,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.9.attn_v.weight",
"offset": 3502366720,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.20.attn_norm.weight",
"offset": 3505250304,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 3505266688,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.20.ffn_up.weight",
"offset": 3545636864,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 3586007040,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.21.attn_norm.weight",
"offset": 3586023424,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 3586039808,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 3634208768,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.21.ffn_up.weight",
"offset": 3674578944,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 3714949120,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 3714965504,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.21.attn_output.weight",
"offset": 3717849088,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.21.attn_q.weight",
"offset": 3729383424,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.21.attn_v.weight",
"offset": 3740917760,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 3744358400,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 3744374784,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 3784744960,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.22.ffn_up.weight",
"offset": 3825115136,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 3865485312,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 3865501696,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.22.attn_output.weight",
"offset": 3868385280,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.22.attn_q.weight",
"offset": 3879919616,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.22.attn_v.weight",
"offset": 3891453952,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.23.attn_norm.weight",
"offset": 3894337536,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 3894353920,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 3934724096,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.23.ffn_up.weight",
"offset": 3975094272,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 4015464448,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 4015480832,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.23.attn_output.weight",
"offset": 4018364416,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.23.attn_q.weight",
"offset": 4029898752,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.23.attn_v.weight",
"offset": 4041433088,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.24.attn_norm.weight",
"offset": 4044316672,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 4044333056,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 4092502016,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.24.ffn_up.weight",
"offset": 4132872192,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 4173242368,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 4173258752,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.24.attn_output.weight",
"offset": 4176142336,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.24.attn_q.weight",
"offset": 4187676672,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.24.attn_v.weight",
"offset": 4199211008,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 4202651648,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 4202668032,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 4243038208,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.25.ffn_up.weight",
"offset": 4283408384,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 4323778560,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 4323794944,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.25.attn_output.weight",
"offset": 4326678528,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.25.attn_q.weight",
"offset": 4338212864,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.25.attn_v.weight",
"offset": 4349747200,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.26.attn_norm.weight",
"offset": 4352630784,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 4352647168,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 4393017344,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.26.ffn_up.weight",
"offset": 4433387520,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 4473757696,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 4473774080,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.26.attn_output.weight",
"offset": 4476657664,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.26.attn_q.weight",
"offset": 4488192000,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.26.attn_v.weight",
"offset": 4499726336,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.27.attn_norm.weight",
"offset": 4502609920,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 4502626304,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 4550795264,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.27.ffn_up.weight",
"offset": 4591165440,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 4631535616,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 4631552000,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.27.attn_output.weight",
"offset": 4634435584,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.27.attn_q.weight",
"offset": 4645969920,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.27.attn_v.weight",
"offset": 4657504256,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 4660944896,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 4660961280,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 4709130240,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.28.ffn_up.weight",
"offset": 4749500416,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 4789870592,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 4789886976,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.28.attn_output.weight",
"offset": 4792770560,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.28.attn_q.weight",
"offset": 4804304896,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.28.attn_v.weight",
"offset": 4815839232,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 4819279872,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 4819296256,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 4867465216,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.29.ffn_up.weight",
"offset": 4907835392,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 4948205568,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 4948221952,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.29.attn_output.weight",
"offset": 4951105536,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.29.attn_q.weight",
"offset": 4962639872,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.29.attn_v.weight",
"offset": 4974174208,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 4977614848,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 4977631232,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 5025800192,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.30.ffn_up.weight",
"offset": 5066170368,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 5106540544,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 5106556928,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.30.attn_output.weight",
"offset": 5109440512,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.30.attn_q.weight",
"offset": 5120974848,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.30.attn_v.weight",
"offset": 5132509184,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 5135949824,
"shape": [
4096,
14336
],
"size": 3440640,
"type": 13
},
{
"name": "blk.31.ffn_up.weight",
"offset": 5176320000,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.31.attn_k.weight",
"offset": 5216690176,
"shape": [
4096,
1024
],
"size": 40370176,
"type": 13
},
{
"name": "blk.31.attn_output.weight",
"offset": 5219573760,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.31.attn_q.weight",
"offset": 5231108096,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.31.attn_v.weight",
"offset": 5242642432,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "output.weight",
"offset": 5246083072,
"shape": [
4096,
128288
],
"size": 3440640,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 5677130752,
"shape": [
4096
],
"size": 431047680,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 5677147136,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 5725316096,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 5725332480,
"shape": [
4096
],
"size": 16384,
"type": 0
}
],
"version": 3
}