a small general purpose model with enticing roleplaying models. Introduce better RAG capabilities in the form of Bagel to Llama 3 8B Instruct as well as German multilanguage, higher general intelligence and vision support.
8B
399 Pulls Updated 7 weeks ago
0d7158670077 · 4.0GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 12,
"general.name": "Llama-3-8B-Ultra-Instruct",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"quantize.imatrix.chunks_count": 88,
"quantize.imatrix.dataset": "/training_data/groups_merged.txt",
"quantize.imatrix.entries_count": 224,
"quantize.imatrix.file": "/models/Llama-3-8B-Ultra-Instruct-GGUF/Llama-3-8B-Ultra-Instruct.imatrix",
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.pre": "llama-bpe",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "token_embd.weight",
"offset": 430940160,
"shape": [
4096,
128256
],
"size": 430940160,
"type": 11
},
{
"name": "blk.0.attn_norm.weight",
"offset": 656670720,
"shape": [
4096
],
"size": 225730560,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 656687104,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000@V\ufffd\u0000\u0000\u0000\u0000",
"offset": 697057280,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 11
},
{
"name": "blk.0.ffn_up.weight",
"offset": 722288640,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 747520000,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 747536384,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.0.attn_output.weight",
"offset": 749338624,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.0.attn_q.weight",
"offset": 758775808,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.0.attn_v.weight",
"offset": 765984768,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 13
},
{
"name": "blk.1.attn_norm.weight",
"offset": 768868352,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 768884736,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 809254912,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 11
},
{
"name": "blk.1.ffn_up.weight",
"offset": 834486272,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 859717632,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 859734016,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.1.attn_output.weight",
"offset": 861536256,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.1.attn_q.weight",
"offset": 870973440,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.1.attn_v.weight",
"offset": 878182400,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 13
},
{
"name": "blk.10.attn_norm.weight",
"offset": 881065984,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 881082368,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 914112512,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.10.ffn_up.weight",
"offset": 939343872,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 964575232,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 964591616,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.10.attn_output.weight",
"offset": 966393856,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.10.attn_q.weight",
"offset": 975831040,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.10.attn_v.weight",
"offset": 983040000,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.11.attn_norm.weight",
"offset": 985399296,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 985415680,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 1018445824,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.11.ffn_up.weight",
"offset": 1043677184,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 1068908544,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 1068924928,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.11.attn_output.weight",
"offset": 1070727168,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.11.attn_q.weight",
"offset": 1080164352,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.11.attn_v.weight",
"offset": 1087373312,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.12.attn_norm.weight",
"offset": 1089732608,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 1089748992,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 1122779136,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.12.ffn_up.weight",
"offset": 1148010496,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 1173241856,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 1173258240,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.12.attn_output.weight",
"offset": 1175060480,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.12.attn_q.weight",
"offset": 1184497664,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.12.attn_v.weight",
"offset": 1191706624,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.13.attn_norm.weight",
"offset": 1194065920,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 1194082304,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 1227112448,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.13.ffn_up.weight",
"offset": 1252343808,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 1277575168,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 1277591552,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.13.attn_output.weight",
"offset": 1279393792,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.13.attn_q.weight",
"offset": 1288830976,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.13.attn_v.weight",
"offset": 1296039936,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.14.attn_norm.weight",
"offset": 1298399232,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 1298415616,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 1331445760,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.14.ffn_up.weight",
"offset": 1356677120,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 1381908480,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 1381924864,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.14.attn_output.weight",
"offset": 1383727104,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.14.attn_q.weight",
"offset": 1393164288,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.14.attn_v.weight",
"offset": 1400373248,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.15.attn_norm.weight",
"offset": 1402732544,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 1402748928,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 1435779072,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.15.ffn_up.weight",
"offset": 1461010432,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 1486241792,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 1486258176,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.15.attn_output.weight",
"offset": 1488060416,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.15.attn_q.weight",
"offset": 1497497600,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.15.attn_v.weight",
"offset": 1504706560,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.16.attn_norm.weight",
"offset": 1507065856,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 1507082240,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 1540112384,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.16.ffn_up.weight",
"offset": 1565343744,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 1590575104,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 1590591488,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.16.attn_output.weight",
"offset": 1592393728,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.16.attn_q.weight",
"offset": 1601830912,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.16.attn_v.weight",
"offset": 1609039872,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.17.attn_norm.weight",
"offset": 1611399168,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 1611415552,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 1644445696,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.17.ffn_up.weight",
"offset": 1669677056,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 1694908416,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 1694924800,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.17.attn_output.weight",
"offset": 1696727040,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.17.attn_q.weight",
"offset": 1706164224,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.17.attn_v.weight",
"offset": 1713373184,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.18.attn_norm.weight",
"offset": 1715732480,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 1715748864,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 1748779008,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.18.ffn_up.weight",
"offset": 1774010368,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 1799241728,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 1799258112,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.18.attn_output.weight",
"offset": 1801060352,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.18.attn_q.weight",
"offset": 1810497536,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.18.attn_v.weight",
"offset": 1817706496,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.19.attn_norm.weight",
"offset": 1820065792,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 1820082176,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 1853112320,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.19.ffn_up.weight",
"offset": 1878343680,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 1903575040,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 1903591424,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.19.attn_output.weight",
"offset": 1905393664,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.19.attn_q.weight",
"offset": 1914830848,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.19.attn_v.weight",
"offset": 1922039808,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.2.attn_norm.weight",
"offset": 1924399104,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 1924415488,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 1957445632,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.2.ffn_up.weight",
"offset": 1982676992,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 2007908352,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 2007924736,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.2.attn_output.weight",
"offset": 2009726976,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.2.attn_q.weight",
"offset": 2019164160,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.2.attn_v.weight",
"offset": 2026373120,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.20.attn_norm.weight",
"offset": 2028732416,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 2028748800,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 2061778944,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.20.ffn_up.weight",
"offset": 2087010304,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 2112241664,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 2112258048,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.20.attn_output.weight",
"offset": 2114060288,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.20.attn_q.weight",
"offset": 2123497472,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.20.attn_v.weight",
"offset": 2130706432,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.21.attn_norm.weight",
"offset": 2133065728,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 2133082112,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 2166112256,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.21.ffn_up.weight",
"offset": 2191343616,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 2216574976,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 2216591360,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.21.attn_output.weight",
"offset": 2218393600,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.21.attn_q.weight",
"offset": 2227830784,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.21.attn_v.weight",
"offset": 2235039744,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.22.attn_norm.weight",
"offset": 2237399040,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 2237415424,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 2270445568,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.22.ffn_up.weight",
"offset": 2295676928,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 2320908288,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 2320924672,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.22.attn_output.weight",
"offset": 2322726912,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.22.attn_q.weight",
"offset": 2332164096,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.22.attn_v.weight",
"offset": 2339373056,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.23.attn_norm.weight",
"offset": 2341732352,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 2341748736,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 2374778880,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.23.ffn_up.weight",
"offset": 2400010240,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 2425241600,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 2425257984,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.23.attn_output.weight",
"offset": 2427060224,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.23.attn_q.weight",
"offset": 2436497408,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.23.attn_v.weight",
"offset": 2443706368,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.24.attn_norm.weight",
"offset": 2446065664,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 2446082048,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 2479112192,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.24.ffn_up.weight",
"offset": 2504343552,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 2529574912,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 2529591296,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.24.attn_output.weight",
"offset": 2531393536,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.24.attn_q.weight",
"offset": 2540830720,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.24.attn_v.weight",
"offset": 2548039680,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.25.attn_norm.weight",
"offset": 2550398976,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 2550415360,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 2583445504,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.25.ffn_up.weight",
"offset": 2608676864,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 2633908224,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 2633924608,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.25.attn_output.weight",
"offset": 2635726848,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.25.attn_q.weight",
"offset": 2645164032,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.25.attn_v.weight",
"offset": 2652372992,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.26.attn_norm.weight",
"offset": 2654732288,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 2654748672,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 2687778816,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.26.ffn_up.weight",
"offset": 2713010176,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 2738241536,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 2738257920,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.26.attn_output.weight",
"offset": 2740060160,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.26.attn_q.weight",
"offset": 2749497344,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.26.attn_v.weight",
"offset": 2756706304,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.27.attn_norm.weight",
"offset": 2759065600,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 2759081984,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 2792112128,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.27.ffn_up.weight",
"offset": 2817343488,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 2842574848,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 2842591232,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.27.attn_output.weight",
"offset": 2844393472,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.27.attn_q.weight",
"offset": 2853830656,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.27.attn_v.weight",
"offset": 2861039616,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.28.attn_norm.weight",
"offset": 2863398912,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 2863415296,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 2896445440,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.28.ffn_up.weight",
"offset": 2921676800,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 2946908160,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 2946924544,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.28.attn_output.weight",
"offset": 2948726784,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.28.attn_q.weight",
"offset": 2958163968,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.28.attn_v.weight",
"offset": 2965372928,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.29.attn_norm.weight",
"offset": 2967732224,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 2967748608,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 3000778752,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.29.ffn_up.weight",
"offset": 3026010112,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 3051241472,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 3051257856,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.29.attn_output.weight",
"offset": 3053060096,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.29.attn_q.weight",
"offset": 3062497280,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.29.attn_v.weight",
"offset": 3069706240,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.3.attn_norm.weight",
"offset": 3072065536,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 3072081920,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 3105112064,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.3.ffn_up.weight",
"offset": 3130343424,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 3155574784,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 3155591168,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.3.attn_output.weight",
"offset": 3157393408,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.3.attn_q.weight",
"offset": 3166830592,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.3.attn_v.weight",
"offset": 3174039552,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.30.attn_norm.weight",
"offset": 3176398848,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 3176415232,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 3209445376,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.30.ffn_up.weight",
"offset": 3234676736,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 3259908096,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 3259924480,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.30.attn_output.weight",
"offset": 3261726720,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.30.attn_q.weight",
"offset": 3271163904,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.30.attn_v.weight",
"offset": 3278372864,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.31.attn_norm.weight",
"offset": 3280732160,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 3280748544,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 3313778688,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.31.ffn_up.weight",
"offset": 3339010048,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 3364241408,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 3364257792,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.31.attn_output.weight",
"offset": 3366060032,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.31.attn_q.weight",
"offset": 3375497216,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.31.attn_v.weight",
"offset": 3382706176,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.4.attn_norm.weight",
"offset": 3385065472,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 3385081856,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 3418112000,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.4.ffn_up.weight",
"offset": 3443343360,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 3468574720,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 3468591104,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.4.attn_output.weight",
"offset": 3470393344,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.4.attn_q.weight",
"offset": 3479830528,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.4.attn_v.weight",
"offset": 3487039488,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.5.attn_norm.weight",
"offset": 3489398784,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 3489415168,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 3522445312,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.5.ffn_up.weight",
"offset": 3547676672,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 3572908032,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 3572924416,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.5.attn_output.weight",
"offset": 3574726656,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.5.attn_q.weight",
"offset": 3584163840,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.5.attn_v.weight",
"offset": 3591372800,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.6.attn_norm.weight",
"offset": 3593732096,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 3593748480,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 3626778624,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.6.ffn_up.weight",
"offset": 3652009984,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 3677241344,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 3677257728,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.6.attn_output.weight",
"offset": 3679059968,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.6.attn_q.weight",
"offset": 3688497152,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.6.attn_v.weight",
"offset": 3695706112,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.7.attn_norm.weight",
"offset": 3698065408,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 3698081792,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 3731111936,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.7.ffn_up.weight",
"offset": 3756343296,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 3781574656,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 3781591040,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.7.attn_output.weight",
"offset": 3783393280,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.7.attn_q.weight",
"offset": 3792830464,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.7.attn_v.weight",
"offset": 3800039424,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.8.attn_norm.weight",
"offset": 3802398720,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 3802415104,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 3835445248,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.8.ffn_up.weight",
"offset": 3860676608,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 3885907968,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 3885924352,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.8.attn_output.weight",
"offset": 3887726592,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "\ufffd\ufffd\u0004\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdi\ufffd\u001a\ufffd\ufffdb\u0019^\ufffd\ufffd",
"offset": 3897163776,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.8.attn_v.weight",
"offset": 3904372736,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "blk.9.attn_norm.weight",
"offset": 3906732032,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 3906748416,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 3939778560,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 11
},
{
"name": "blk.9.ffn_up.weight",
"offset": 3965009920,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 11
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 3990241280,
"shape": [
4096
],
"size": 25231360,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 3990257664,
"shape": [
4096,
1024
],
"size": 16384,
"type": 11
},
{
"name": "blk.9.attn_output.weight",
"offset": 3992059904,
"shape": [
4096,
4096
],
"size": 1802240,
"type": 12
},
{
"name": "blk.9.attn_q.weight",
"offset": 4001497088,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.9.attn_v.weight",
"offset": 4008706048,
"shape": [
4096,
1024
],
"size": 7208960,
"type": 12
},
{
"name": "output_norm.weight",
"offset": 4011065344,
"shape": [
4096
],
"size": 2359296,
"type": 0
}
],
"version": 3
}