a small general purpose model with enticing roleplaying models. Introduce better RAG capabilities in the form of Bagel to Llama 3 8B Instruct as well as German multilanguage, higher general intelligence and vision support.
8B
397 Pulls Updated 7 weeks ago
0e25a42f835f · 3.2GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 10,
"general.name": "Llama-3-8B-Ultra-Instruct",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"quantize.imatrix.chunks_count": 88,
"quantize.imatrix.dataset": "/training_data/groups_merged.txt",
"quantize.imatrix.entries_count": 224,
"quantize.imatrix.file": "/models/Llama-3-8B-Ultra-Instruct-GGUF/Llama-3-8B-Ultra-Instruct.imatrix",
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.pre": "llama-bpe",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "token_embd.weight",
"offset": 430940160,
"shape": [
4096,
128256
],
"size": 430940160,
"type": 10
},
{
"name": "blk.0.attn_norm.weight",
"offset": 603316224,
"shape": [
4096
],
"size": 172376064,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 603332608,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 628563968,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.0.ffn_up.weight",
"offset": 647831552,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 667099136,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 667115520,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.0.attn_output.weight",
"offset": 668491776,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.0.attn_q.weight",
"offset": 675700736,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.0.attn_v.weight",
"offset": 681205760,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.1.attn_norm.weight",
"offset": 683565056,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 683581440,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 708812800,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.1.ffn_up.weight",
"offset": 728080384,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 747347968,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 747364352,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.1.attn_output.weight",
"offset": 748740608,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.1.attn_q.weight",
"offset": 755949568,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.1.attn_v.weight",
"offset": 761454592,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.10.attn_norm.weight",
"offset": 763813888,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 763830272,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 789061632,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.10.ffn_up.weight",
"offset": 808329216,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 827596800,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 827613184,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.10.attn_output.weight",
"offset": 828989440,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.10.attn_q.weight",
"offset": 836198400,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.10.attn_v.weight",
"offset": 841703424,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.11.attn_norm.weight",
"offset": 844062720,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 844079104,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 869310464,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.11.ffn_up.weight",
"offset": 888578048,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 907845632,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 907862016,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.11.attn_output.weight",
"offset": 909238272,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.11.attn_q.weight",
"offset": 916447232,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.11.attn_v.weight",
"offset": 921952256,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.12.attn_norm.weight",
"offset": 924311552,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 924327936,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 949559296,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.12.ffn_up.weight",
"offset": 968826880,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 988094464,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 988110848,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.12.attn_output.weight",
"offset": 989487104,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.12.attn_q.weight",
"offset": 996696064,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.12.attn_v.weight",
"offset": 1002201088,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.13.attn_norm.weight",
"offset": 1004560384,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 1004576768,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 1029808128,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.13.ffn_up.weight",
"offset": 1049075712,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 1068343296,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 1068359680,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.13.attn_output.weight",
"offset": 1069735936,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.13.attn_q.weight",
"offset": 1076944896,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.13.attn_v.weight",
"offset": 1082449920,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.14.attn_norm.weight",
"offset": 1084809216,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 1084825600,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 1110056960,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.14.ffn_up.weight",
"offset": 1129324544,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 1148592128,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 1148608512,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.14.attn_output.weight",
"offset": 1149984768,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.14.attn_q.weight",
"offset": 1157193728,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.14.attn_v.weight",
"offset": 1162698752,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.15.attn_norm.weight",
"offset": 1165058048,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 1165074432,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 1190305792,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.15.ffn_up.weight",
"offset": 1209573376,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 1228840960,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 1228857344,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.15.attn_output.weight",
"offset": 1230233600,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.15.attn_q.weight",
"offset": 1237442560,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.15.attn_v.weight",
"offset": 1242947584,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.16.attn_norm.weight",
"offset": 1245306880,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 1245323264,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 1270554624,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.16.ffn_up.weight",
"offset": 1289822208,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 1309089792,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 1309106176,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.16.attn_output.weight",
"offset": 1310482432,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.16.attn_q.weight",
"offset": 1317691392,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.16.attn_v.weight",
"offset": 1323196416,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.17.attn_norm.weight",
"offset": 1325555712,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 1325572096,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 1350803456,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.17.ffn_up.weight",
"offset": 1370071040,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 1389338624,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 1389355008,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.17.attn_output.weight",
"offset": 1390731264,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.17.attn_q.weight",
"offset": 1397940224,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.17.attn_v.weight",
"offset": 1403445248,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.18.attn_norm.weight",
"offset": 1405804544,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 1405820928,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 1431052288,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.18.ffn_up.weight",
"offset": 1450319872,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 1469587456,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 1469603840,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.18.attn_output.weight",
"offset": 1470980096,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.18.attn_q.weight",
"offset": 1478189056,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.18.attn_v.weight",
"offset": 1483694080,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.19.attn_norm.weight",
"offset": 1486053376,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 1486069760,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 1511301120,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.19.ffn_up.weight",
"offset": 1530568704,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 1549836288,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 1549852672,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.19.attn_output.weight",
"offset": 1551228928,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.19.attn_q.weight",
"offset": 1558437888,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.19.attn_v.weight",
"offset": 1563942912,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.2.attn_norm.weight",
"offset": 1566302208,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 1566318592,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 1591549952,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.2.ffn_up.weight",
"offset": 1610817536,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 1630085120,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 1630101504,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.2.attn_output.weight",
"offset": 1631477760,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.2.attn_q.weight",
"offset": 1638686720,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.2.attn_v.weight",
"offset": 1644191744,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.20.attn_norm.weight",
"offset": 1646551040,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 1646567424,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 1671798784,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.20.ffn_up.weight",
"offset": 1691066368,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 1710333952,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 1710350336,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.20.attn_output.weight",
"offset": 1711726592,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.20.attn_q.weight",
"offset": 1718935552,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.20.attn_v.weight",
"offset": 1724440576,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.21.attn_norm.weight",
"offset": 1726799872,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 1726816256,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 1752047616,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.21.ffn_up.weight",
"offset": 1771315200,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 1790582784,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 1790599168,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.21.attn_output.weight",
"offset": 1791975424,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.21.attn_q.weight",
"offset": 1799184384,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.21.attn_v.weight",
"offset": 1804689408,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.22.attn_norm.weight",
"offset": 1807048704,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 1807065088,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 1832296448,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.22.ffn_up.weight",
"offset": 1851564032,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 1870831616,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 1870848000,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.22.attn_output.weight",
"offset": 1872224256,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.22.attn_q.weight",
"offset": 1879433216,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.22.attn_v.weight",
"offset": 1884938240,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.23.attn_norm.weight",
"offset": 1887297536,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 1887313920,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 1912545280,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.23.ffn_up.weight",
"offset": 1931812864,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 1951080448,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 1951096832,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.23.attn_output.weight",
"offset": 1952473088,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.23.attn_q.weight",
"offset": 1959682048,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.23.attn_v.weight",
"offset": 1965187072,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.24.attn_norm.weight",
"offset": 1967546368,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 1967562752,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 1992794112,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.24.ffn_up.weight",
"offset": 2012061696,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 2031329280,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 2031345664,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.24.attn_output.weight",
"offset": 2032721920,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.24.attn_q.weight",
"offset": 2039930880,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.24.attn_v.weight",
"offset": 2045435904,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.25.attn_norm.weight",
"offset": 2047795200,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 2047811584,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 2073042944,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.25.ffn_up.weight",
"offset": 2092310528,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 2111578112,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 2111594496,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.25.attn_output.weight",
"offset": 2112970752,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.25.attn_q.weight",
"offset": 2120179712,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.25.attn_v.weight",
"offset": 2125684736,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.26.attn_norm.weight",
"offset": 2128044032,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 2128060416,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 2153291776,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.26.ffn_up.weight",
"offset": 2172559360,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 2191826944,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 2191843328,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.26.attn_output.weight",
"offset": 2193219584,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.26.attn_q.weight",
"offset": 2200428544,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.26.attn_v.weight",
"offset": 2205933568,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.27.attn_norm.weight",
"offset": 2208292864,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 2208309248,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 2233540608,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.27.ffn_up.weight",
"offset": 2252808192,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 2272075776,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 2272092160,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.27.attn_output.weight",
"offset": 2273468416,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.27.attn_q.weight",
"offset": 2280677376,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.27.attn_v.weight",
"offset": 2286182400,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.28.attn_norm.weight",
"offset": 2288541696,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 2288558080,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 2313789440,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.28.ffn_up.weight",
"offset": 2333057024,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 2352324608,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 2352340992,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.28.attn_output.weight",
"offset": 2353717248,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.28.attn_q.weight",
"offset": 2360926208,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.28.attn_v.weight",
"offset": 2366431232,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.29.attn_norm.weight",
"offset": 2368790528,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 2368806912,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 2394038272,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.29.ffn_up.weight",
"offset": 2413305856,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 2432573440,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 2432589824,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.29.attn_output.weight",
"offset": 2433966080,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.29.attn_q.weight",
"offset": 2441175040,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.29.attn_v.weight",
"offset": 2446680064,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.3.attn_norm.weight",
"offset": 2449039360,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 2449055744,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 2474287104,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.3.ffn_up.weight",
"offset": 2493554688,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 2512822272,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 2512838656,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.3.attn_output.weight",
"offset": 2514214912,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.3.attn_q.weight",
"offset": 2521423872,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.3.attn_v.weight",
"offset": 2526928896,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.30.attn_norm.weight",
"offset": 2529288192,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 2529304576,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 2554535936,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.30.ffn_up.weight",
"offset": 2573803520,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 2593071104,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 2593087488,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.30.attn_output.weight",
"offset": 2594463744,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.30.attn_q.weight",
"offset": 2601672704,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.30.attn_v.weight",
"offset": 2607177728,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.31.attn_norm.weight",
"offset": 2609537024,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 2609553408,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 2634784768,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.31.ffn_up.weight",
"offset": 2654052352,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 2673319936,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 2673336320,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.31.attn_output.weight",
"offset": 2674712576,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.31.attn_q.weight",
"offset": 2681921536,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.31.attn_v.weight",
"offset": 2687426560,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.4.attn_norm.weight",
"offset": 2689785856,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 2689802240,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 2715033600,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.4.ffn_up.weight",
"offset": 2734301184,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 2753568768,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 2753585152,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.4.attn_output.weight",
"offset": 2754961408,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.4.attn_q.weight",
"offset": 2762170368,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.4.attn_v.weight",
"offset": 2767675392,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.5.attn_norm.weight",
"offset": 2770034688,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 2770051072,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 2795282432,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.5.ffn_up.weight",
"offset": 2814550016,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 2833817600,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 2833833984,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.5.attn_output.weight",
"offset": 2835210240,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.5.attn_q.weight",
"offset": 2842419200,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.5.attn_v.weight",
"offset": 2847924224,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.6.attn_norm.weight",
"offset": 2850283520,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 2850299904,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 2875531264,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.6.ffn_up.weight",
"offset": 2894798848,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 2914066432,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 2914082816,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.6.attn_output.weight",
"offset": 2915459072,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.6.attn_q.weight",
"offset": 2922668032,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.6.attn_v.weight",
"offset": 2928173056,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.7.attn_norm.weight",
"offset": 2930532352,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 2930548736,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 2955780096,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.7.ffn_up.weight",
"offset": 2975047680,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 2994315264,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 2994331648,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.7.attn_output.weight",
"offset": 2995707904,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.7.attn_q.weight",
"offset": 3002916864,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.7.attn_v.weight",
"offset": 3008421888,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.8.attn_norm.weight",
"offset": 3010781184,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 3010797568,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 3036028928,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.8.ffn_up.weight",
"offset": 3055296512,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 3074564096,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 3074580480,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.8.attn_output.weight",
"offset": 3075956736,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.8.attn_q.weight",
"offset": 3083165696,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.8.attn_v.weight",
"offset": 3088670720,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "blk.9.attn_norm.weight",
"offset": 3091030016,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 3091046400,
"shape": [
14336,
4096
],
"size": 16384,
"type": 11
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 3116277760,
"shape": [
4096,
14336
],
"size": 25231360,
"type": 10
},
{
"name": "blk.9.ffn_up.weight",
"offset": 3135545344,
"shape": [
4096,
14336
],
"size": 19267584,
"type": 10
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 3154812928,
"shape": [
4096
],
"size": 19267584,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 3154829312,
"shape": [
4096,
1024
],
"size": 16384,
"type": 10
},
{
"name": "blk.9.attn_output.weight",
"offset": 3156205568,
"shape": [
4096,
4096
],
"size": 1376256,
"type": 11
},
{
"name": "blk.9.attn_q.weight",
"offset": 3163414528,
"shape": [
4096,
4096
],
"size": 7208960,
"type": 10
},
{
"name": "blk.9.attn_v.weight",
"offset": 3168919552,
"shape": [
4096,
1024
],
"size": 5505024,
"type": 12
},
{
"name": "output_norm.weight",
"offset": 3171278848,
"shape": [
4096
],
"size": 2359296,
"type": 0
}
],
"version": 3
}