This new version of Hermes maintains its excellent general task and conversation capabilities - but also excels at Function Calling, JSON Structured Outputs, and has improved on several other metrics as well
8B
177 Pulls Updated 8 weeks ago
d138388cfda0 · 8.5GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 7,
"general.name": "Hermes-2-Pro-Llama-3-8B",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128288,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128003,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.pre": "llama-bpe",
"tokenizer.ggml.token_type": "... (128288 values)",
"tokenizer.ggml.tokens": "... (128288 values)"
},
"num_params": 8030523392,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 558309376,
"shape": [
4096
],
"size": 558309376,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 558325760,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 620716032,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.0.ffn_up.weight",
"offset": 683106304,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 745496576,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 745512960,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.0.attn_output.weight",
"offset": 749969408,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.0.attn_q.weight",
"offset": 767795200,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.0.attn_v.weight",
"offset": 785620992,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.1.attn_norm.weight",
"offset": 790077440,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 790093824,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 852484096,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.1.ffn_up.weight",
"offset": 914874368,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 977264640,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 977281024,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.1.attn_output.weight",
"offset": 981737472,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.1.attn_q.weight",
"offset": 999563264,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.1.attn_v.weight",
"offset": 1017389056,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.2.attn_norm.weight",
"offset": 1021845504,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 1021861888,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 1084252160,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.2.ffn_up.weight",
"offset": 1146642432,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 1209032704,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 1209049088,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.2.attn_output.weight",
"offset": 1213505536,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.2.attn_q.weight",
"offset": 1231331328,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.2.attn_v.weight",
"offset": 1249157120,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.3.attn_norm.weight",
"offset": 1253613568,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 1253629952,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 1316020224,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.3.ffn_up.weight",
"offset": 1378410496,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 1440800768,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 1440817152,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.3.attn_output.weight",
"offset": 1445273600,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.3.attn_q.weight",
"offset": 1463099392,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.3.attn_v.weight",
"offset": 1480925184,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.4.attn_norm.weight",
"offset": 1485381632,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 1485398016,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 1547788288,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1610178560,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1672568832,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 1672585216,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.4.attn_output.weight",
"offset": 1677041664,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.4.attn_q.weight",
"offset": 1694867456,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.4.attn_v.weight",
"offset": 1712693248,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.5.attn_norm.weight",
"offset": 1717149696,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 1717166080,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1779556352,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1841946624,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1904336896,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1904353280,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.5.attn_output.weight",
"offset": 1908809728,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.5.attn_q.weight",
"offset": 1926635520,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.5.attn_v.weight",
"offset": 1944461312,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1948917760,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1948934144,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 2011324416,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.6.ffn_up.weight",
"offset": 2073714688,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 2136104960,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 2136121344,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.6.attn_output.weight",
"offset": 2140577792,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.6.attn_q.weight",
"offset": 2158403584,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.6.attn_v.weight",
"offset": 2176229376,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.7.attn_norm.weight",
"offset": 2180685824,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 2180702208,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 2243092480,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.7.ffn_up.weight",
"offset": 2305482752,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 2367873024,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 2367889408,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.7.attn_output.weight",
"offset": 2372345856,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.7.attn_q.weight",
"offset": 2390171648,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.7.attn_v.weight",
"offset": 2407997440,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.8.attn_norm.weight",
"offset": 2412453888,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 2412470272,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 2474860544,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.8.ffn_up.weight",
"offset": 2537250816,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 2599641088,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 2599657472,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.8.attn_output.weight",
"offset": 2604113920,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.8.attn_q.weight",
"offset": 2621939712,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.8.attn_v.weight",
"offset": 2639765504,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.10.attn_norm.weight",
"offset": 2644221952,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 2644238336,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 2706628608,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.10.ffn_up.weight",
"offset": 2769018880,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 2831409152,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 2831425536,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.10.attn_output.weight",
"offset": 2835881984,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.10.attn_q.weight",
"offset": 2853707776,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.10.attn_v.weight",
"offset": 2871533568,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.11.attn_norm.weight",
"offset": 2875990016,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 2876006400,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 2938396672,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.11.ffn_up.weight",
"offset": 3000786944,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 3063177216,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 3063193600,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.11.attn_output.weight",
"offset": 3067650048,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.11.attn_q.weight",
"offset": 3085475840,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.11.attn_v.weight",
"offset": 3103301632,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.12.attn_norm.weight",
"offset": 3107758080,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 3107774464,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 3170164736,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.12.ffn_up.weight",
"offset": 3232555008,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 3294945280,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 3294961664,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.12.attn_output.weight",
"offset": 3299418112,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.12.attn_q.weight",
"offset": 3317243904,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.12.attn_v.weight",
"offset": 3335069696,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.13.attn_norm.weight",
"offset": 3339526144,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 3339542528,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 3401932800,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.13.ffn_up.weight",
"offset": 3464323072,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 3526713344,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 3526729728,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.13.attn_output.weight",
"offset": 3531186176,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.13.attn_q.weight",
"offset": 3549011968,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.13.attn_v.weight",
"offset": 3566837760,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.14.attn_norm.weight",
"offset": 3571294208,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 3571310592,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 3633700864,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.14.ffn_up.weight",
"offset": 3696091136,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 3758481408,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 3758497792,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.14.attn_output.weight",
"offset": 3762954240,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.14.attn_q.weight",
"offset": 3780780032,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.14.attn_v.weight",
"offset": 3798605824,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.15.attn_norm.weight",
"offset": 3803062272,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 3803078656,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 3865468928,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.15.ffn_up.weight",
"offset": 3927859200,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 3990249472,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 3990265856,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.15.attn_output.weight",
"offset": 3994722304,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.15.attn_q.weight",
"offset": 4012548096,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.15.attn_v.weight",
"offset": 4030373888,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.16.attn_norm.weight",
"offset": 4034830336,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 4034846720,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 4097236992,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.16.ffn_up.weight",
"offset": 4159627264,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 4222017536,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 4222033920,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.16.attn_output.weight",
"offset": 4226490368,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.16.attn_q.weight",
"offset": 4244316160,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.16.attn_v.weight",
"offset": 4262141952,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.17.attn_norm.weight",
"offset": 4266598400,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 4266614784,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 4329005056,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.17.ffn_up.weight",
"offset": 4391395328,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 4453785600,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 4453801984,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.17.attn_output.weight",
"offset": 4458258432,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.17.attn_q.weight",
"offset": 4476084224,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.17.attn_v.weight",
"offset": 4493910016,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.18.attn_norm.weight",
"offset": 4498366464,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 4498382848,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 4560773120,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.18.ffn_up.weight",
"offset": 4623163392,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 4685553664,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 4685570048,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.18.attn_output.weight",
"offset": 4690026496,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.18.attn_q.weight",
"offset": 4707852288,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.18.attn_v.weight",
"offset": 4725678080,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.19.attn_norm.weight",
"offset": 4730134528,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 4730150912,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 4792541184,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.19.ffn_up.weight",
"offset": 4854931456,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 4917321728,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 4917338112,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.19.attn_output.weight",
"offset": 4921794560,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.19.attn_q.weight",
"offset": 4939620352,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.19.attn_v.weight",
"offset": 4957446144,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 4961902592,
"shape": [
4096,
14336
],
"size": 4456448,
"type": 8
},
{
"name": "blk.20.attn_k.weight",
"offset": 5024292864,
"shape": [
4096,
1024
],
"size": 62390272,
"type": 8
},
{
"name": "blk.20.attn_output.weight",
"offset": 5028749312,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.20.attn_q.weight",
"offset": 5046575104,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.20.attn_v.weight",
"offset": 5064400896,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.9.attn_norm.weight",
"offset": 5068857344,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 5068873728,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 5131264000,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.9.ffn_up.weight",
"offset": 5193654272,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 5256044544,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 5256060928,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.9.attn_output.weight",
"offset": 5260517376,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.9.attn_q.weight",
"offset": 5278343168,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.9.attn_v.weight",
"offset": 5296168960,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.20.attn_norm.weight",
"offset": 5300625408,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 5300641792,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.20.ffn_up.weight",
"offset": 5363032064,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 5425422336,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.21.attn_norm.weight",
"offset": 5425438720,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 5425455104,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 5487845376,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.21.ffn_up.weight",
"offset": 5550235648,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 5612625920,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 5612642304,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.21.attn_output.weight",
"offset": 5617098752,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.21.attn_q.weight",
"offset": 5634924544,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.21.attn_v.weight",
"offset": 5652750336,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.22.attn_norm.weight",
"offset": 5657206784,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 5657223168,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 5719613440,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.22.ffn_up.weight",
"offset": 5782003712,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 5844393984,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 5844410368,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.22.attn_output.weight",
"offset": 5848866816,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.22.attn_q.weight",
"offset": 5866692608,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.22.attn_v.weight",
"offset": 5884518400,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.23.attn_norm.weight",
"offset": 5888974848,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 5888991232,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 5951381504,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.23.ffn_up.weight",
"offset": 6013771776,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 6076162048,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 6076178432,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.23.attn_output.weight",
"offset": 6080634880,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.23.attn_q.weight",
"offset": 6098460672,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.23.attn_v.weight",
"offset": 6116286464,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.24.attn_norm.weight",
"offset": 6120742912,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 6120759296,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 6183149568,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.24.ffn_up.weight",
"offset": 6245539840,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 6307930112,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 6307946496,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.24.attn_output.weight",
"offset": 6312402944,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.24.attn_q.weight",
"offset": 6330228736,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.24.attn_v.weight",
"offset": 6348054528,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.25.attn_norm.weight",
"offset": 6352510976,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 6352527360,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 6414917632,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.25.ffn_up.weight",
"offset": 6477307904,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 6539698176,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 6539714560,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.25.attn_output.weight",
"offset": 6544171008,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.25.attn_q.weight",
"offset": 6561996800,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.25.attn_v.weight",
"offset": 6579822592,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.26.attn_norm.weight",
"offset": 6584279040,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 6584295424,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 6646685696,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.26.ffn_up.weight",
"offset": 6709075968,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 6771466240,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 6771482624,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.26.attn_output.weight",
"offset": 6775939072,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.26.attn_q.weight",
"offset": 6793764864,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.26.attn_v.weight",
"offset": 6811590656,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.27.attn_norm.weight",
"offset": 6816047104,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 6816063488,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 6878453760,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.27.ffn_up.weight",
"offset": 6940844032,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 7003234304,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 7003250688,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.27.attn_output.weight",
"offset": 7007707136,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.27.attn_q.weight",
"offset": 7025532928,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.27.attn_v.weight",
"offset": 7043358720,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.28.attn_norm.weight",
"offset": 7047815168,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 7047831552,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 7110221824,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.28.ffn_up.weight",
"offset": 7172612096,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 7235002368,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 7235018752,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.28.attn_output.weight",
"offset": 7239475200,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.28.attn_q.weight",
"offset": 7257300992,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.28.attn_v.weight",
"offset": 7275126784,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.29.attn_norm.weight",
"offset": 7279583232,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 7279599616,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 7341989888,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.29.ffn_up.weight",
"offset": 7404380160,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 7466770432,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 7466786816,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.29.attn_output.weight",
"offset": 7471243264,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.29.attn_q.weight",
"offset": 7489069056,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.29.attn_v.weight",
"offset": 7506894848,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.30.attn_norm.weight",
"offset": 7511351296,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 7511367680,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 7573757952,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.30.ffn_up.weight",
"offset": 7636148224,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 7698538496,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 7698554880,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.30.attn_output.weight",
"offset": 7703011328,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.30.attn_q.weight",
"offset": 7720837120,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.30.attn_v.weight",
"offset": 7738662912,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 7743119360,
"shape": [
4096,
14336
],
"size": 4456448,
"type": 8
},
{
"name": "blk.31.ffn_up.weight",
"offset": 7805509632,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.31.attn_k.weight",
"offset": 7867899904,
"shape": [
4096,
1024
],
"size": 62390272,
"type": 8
},
{
"name": "blk.31.attn_output.weight",
"offset": 7872356352,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.31.attn_q.weight",
"offset": 7890182144,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.31.attn_v.weight",
"offset": 7908007936,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "output.weight",
"offset": 7912464384,
"shape": [
4096,
128288
],
"size": 4456448,
"type": 8
},
{
"name": "blk.31.attn_norm.weight",
"offset": 8470773760,
"shape": [
4096
],
"size": 558309376,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 8470790144,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 8533180416,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 8533196800,
"shape": [
4096
],
"size": 16384,
"type": 0
}
],
"version": 3
}