a small general purpose model with enticing roleplaying models. Introduce better RAG capabilities in the form of Bagel to Llama 3 8B Instruct as well as German multilanguage, higher general intelligence and vision support.
8B
400 Pulls Updated 7 weeks ago
20f27ba43492 · 8.5GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 7,
"general.name": "Llama-3-8B-Ultra-Instruct",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"quantize.imatrix.chunks_count": 88,
"quantize.imatrix.dataset": "/training_data/groups_merged.txt",
"quantize.imatrix.entries_count": 224,
"quantize.imatrix.file": "/models/Llama-3-8B-Ultra-Instruct-GGUF/Llama-3-8B-Ultra-Instruct.imatrix",
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.pre": "llama-bpe",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "token_embd.weight",
"offset": 558170112,
"shape": [
4096,
128256
],
"size": 558170112,
"type": 8
},
{
"name": "blk.0.attn_norm.weight",
"offset": 1116340224,
"shape": [
4096
],
"size": 558170112,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 1116356608,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\ufffd\u001c\u0001\u0000\u0000\u0000",
"offset": 1178746880,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.0.ffn_up.weight",
"offset": 1241137152,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 1303527424,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 1303543808,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.0.attn_output.weight",
"offset": 1308000256,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.0.attn_q.weight",
"offset": 1325826048,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.0.attn_v.weight",
"offset": 1343651840,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.1.attn_norm.weight",
"offset": 1348108288,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 1348124672,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 1410514944,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.1.ffn_up.weight",
"offset": 1472905216,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 1535295488,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 1535311872,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.1.attn_output.weight",
"offset": 1539768320,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.1.attn_q.weight",
"offset": 1557594112,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.1.attn_v.weight",
"offset": 1575419904,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.10.attn_norm.weight",
"offset": 1579876352,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 1579892736,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 1642283008,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.10.ffn_up.weight",
"offset": 1704673280,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 1767063552,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 1767079936,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.10.attn_output.weight",
"offset": 1771536384,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.10.attn_q.weight",
"offset": 1789362176,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.10.attn_v.weight",
"offset": 1807187968,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.11.attn_norm.weight",
"offset": 1811644416,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 1811660800,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 1874051072,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.11.ffn_up.weight",
"offset": 1936441344,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 1998831616,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 1998848000,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.11.attn_output.weight",
"offset": 2003304448,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.11.attn_q.weight",
"offset": 2021130240,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.11.attn_v.weight",
"offset": 2038956032,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.12.attn_norm.weight",
"offset": 2043412480,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 2043428864,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 2105819136,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.12.ffn_up.weight",
"offset": 2168209408,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 2230599680,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 2230616064,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.12.attn_output.weight",
"offset": 2235072512,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.12.attn_q.weight",
"offset": 2252898304,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.12.attn_v.weight",
"offset": 2270724096,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.13.attn_norm.weight",
"offset": 2275180544,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 2275196928,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 2337587200,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.13.ffn_up.weight",
"offset": 2399977472,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 2462367744,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 2462384128,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.13.attn_output.weight",
"offset": 2466840576,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.13.attn_q.weight",
"offset": 2484666368,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.13.attn_v.weight",
"offset": 2502492160,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.14.attn_norm.weight",
"offset": 2506948608,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 2506964992,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 2569355264,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.14.ffn_up.weight",
"offset": 2631745536,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 2694135808,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 2694152192,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.14.attn_output.weight",
"offset": 2698608640,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.14.attn_q.weight",
"offset": 2716434432,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.14.attn_v.weight",
"offset": 2734260224,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.15.attn_norm.weight",
"offset": 2738716672,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 2738733056,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 2801123328,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.15.ffn_up.weight",
"offset": 2863513600,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 2925903872,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 2925920256,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.15.attn_output.weight",
"offset": 2930376704,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.15.attn_q.weight",
"offset": 2948202496,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.15.attn_v.weight",
"offset": 2966028288,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.16.attn_norm.weight",
"offset": 2970484736,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 2970501120,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 3032891392,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.16.ffn_up.weight",
"offset": 3095281664,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 3157671936,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 3157688320,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.16.attn_output.weight",
"offset": 3162144768,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.16.attn_q.weight",
"offset": 3179970560,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.16.attn_v.weight",
"offset": 3197796352,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.17.attn_norm.weight",
"offset": 3202252800,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 3202269184,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 3264659456,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.17.ffn_up.weight",
"offset": 3327049728,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 3389440000,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 3389456384,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.17.attn_output.weight",
"offset": 3393912832,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.17.attn_q.weight",
"offset": 3411738624,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.17.attn_v.weight",
"offset": 3429564416,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.18.attn_norm.weight",
"offset": 3434020864,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 3434037248,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 3496427520,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.18.ffn_up.weight",
"offset": 3558817792,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 3621208064,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 3621224448,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.18.attn_output.weight",
"offset": 3625680896,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.18.attn_q.weight",
"offset": 3643506688,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.18.attn_v.weight",
"offset": 3661332480,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.19.attn_norm.weight",
"offset": 3665788928,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 3665805312,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 3728195584,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.19.ffn_up.weight",
"offset": 3790585856,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 3852976128,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 3852992512,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.19.attn_output.weight",
"offset": 3857448960,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.19.attn_q.weight",
"offset": 3875274752,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.19.attn_v.weight",
"offset": 3893100544,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.2.attn_norm.weight",
"offset": 3897556992,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 3897573376,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 3959963648,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.2.ffn_up.weight",
"offset": 4022353920,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 4084744192,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 4084760576,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.2.attn_output.weight",
"offset": 4089217024,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.2.attn_q.weight",
"offset": 4107042816,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.2.attn_v.weight",
"offset": 4124868608,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.20.attn_norm.weight",
"offset": 4129325056,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 4129341440,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 4191731712,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.20.ffn_up.weight",
"offset": 4254121984,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 4316512256,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 4316528640,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.20.attn_output.weight",
"offset": 4320985088,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.20.attn_q.weight",
"offset": 4338810880,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.20.attn_v.weight",
"offset": 4356636672,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.21.attn_norm.weight",
"offset": 4361093120,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 4361109504,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 4423499776,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.21.ffn_up.weight",
"offset": 4485890048,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 4548280320,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 4548296704,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.21.attn_output.weight",
"offset": 4552753152,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.21.attn_q.weight",
"offset": 4570578944,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.21.attn_v.weight",
"offset": 4588404736,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.22.attn_norm.weight",
"offset": 4592861184,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 4592877568,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 4655267840,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.22.ffn_up.weight",
"offset": 4717658112,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 4780048384,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 4780064768,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.22.attn_output.weight",
"offset": 4784521216,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.22.attn_q.weight",
"offset": 4802347008,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.22.attn_v.weight",
"offset": 4820172800,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.23.attn_norm.weight",
"offset": 4824629248,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 4824645632,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 4887035904,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.23.ffn_up.weight",
"offset": 4949426176,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 5011816448,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 5011832832,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.23.attn_output.weight",
"offset": 5016289280,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.23.attn_q.weight",
"offset": 5034115072,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.23.attn_v.weight",
"offset": 5051940864,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.24.attn_norm.weight",
"offset": 5056397312,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 5056413696,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 5118803968,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.24.ffn_up.weight",
"offset": 5181194240,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 5243584512,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 5243600896,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.24.attn_output.weight",
"offset": 5248057344,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.24.attn_q.weight",
"offset": 5265883136,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.24.attn_v.weight",
"offset": 5283708928,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.25.attn_norm.weight",
"offset": 5288165376,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 5288181760,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 5350572032,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.25.ffn_up.weight",
"offset": 5412962304,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 5475352576,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 5475368960,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.25.attn_output.weight",
"offset": 5479825408,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.25.attn_q.weight",
"offset": 5497651200,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.25.attn_v.weight",
"offset": 5515476992,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.26.attn_norm.weight",
"offset": 5519933440,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 5519949824,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 5582340096,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.26.ffn_up.weight",
"offset": 5644730368,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 5707120640,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 5707137024,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.26.attn_output.weight",
"offset": 5711593472,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.26.attn_q.weight",
"offset": 5729419264,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.26.attn_v.weight",
"offset": 5747245056,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.27.attn_norm.weight",
"offset": 5751701504,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 5751717888,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 5814108160,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.27.ffn_up.weight",
"offset": 5876498432,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 5938888704,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 5938905088,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.27.attn_output.weight",
"offset": 5943361536,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.27.attn_q.weight",
"offset": 5961187328,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.27.attn_v.weight",
"offset": 5979013120,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.28.attn_norm.weight",
"offset": 5983469568,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 5983485952,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 6045876224,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.28.ffn_up.weight",
"offset": 6108266496,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 6170656768,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 6170673152,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.28.attn_output.weight",
"offset": 6175129600,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.28.attn_q.weight",
"offset": 6192955392,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.28.attn_v.weight",
"offset": 6210781184,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.29.attn_norm.weight",
"offset": 6215237632,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 6215254016,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 6277644288,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.29.ffn_up.weight",
"offset": 6340034560,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 6402424832,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 6402441216,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.29.attn_output.weight",
"offset": 6406897664,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.29.attn_q.weight",
"offset": 6424723456,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.29.attn_v.weight",
"offset": 6442549248,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.3.attn_norm.weight",
"offset": 6447005696,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 6447022080,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 6509412352,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.3.ffn_up.weight",
"offset": 6571802624,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 6634192896,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 6634209280,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.3.attn_output.weight",
"offset": 6638665728,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.3.attn_q.weight",
"offset": 6656491520,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.3.attn_v.weight",
"offset": 6674317312,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.30.attn_norm.weight",
"offset": 6678773760,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 6678790144,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 6741180416,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.30.ffn_up.weight",
"offset": 6803570688,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 6865960960,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 6865977344,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.30.attn_output.weight",
"offset": 6870433792,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.30.attn_q.weight",
"offset": 6888259584,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.30.attn_v.weight",
"offset": 6906085376,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.31.attn_norm.weight",
"offset": 6910541824,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 6910558208,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 6972948480,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.31.ffn_up.weight",
"offset": 7035338752,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 7097729024,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 7097745408,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.31.attn_output.weight",
"offset": 7102201856,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.31.attn_q.weight",
"offset": 7120027648,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.31.attn_v.weight",
"offset": 7137853440,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.4.attn_norm.weight",
"offset": 7142309888,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 7142326272,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 7204716544,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.4.ffn_up.weight",
"offset": 7267106816,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 7329497088,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 7329513472,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.4.attn_output.weight",
"offset": 7333969920,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.4.attn_q.weight",
"offset": 7351795712,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.4.attn_v.weight",
"offset": 7369621504,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.5.attn_norm.weight",
"offset": 7374077952,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 7374094336,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 7436484608,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.5.ffn_up.weight",
"offset": 7498874880,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 7561265152,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 7561281536,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.5.attn_output.weight",
"offset": 7565737984,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.5.attn_q.weight",
"offset": 7583563776,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.5.attn_v.weight",
"offset": 7601389568,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.6.attn_norm.weight",
"offset": 7605846016,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 7605862400,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 7668252672,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.6.ffn_up.weight",
"offset": 7730642944,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 7793033216,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 7793049600,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.6.attn_output.weight",
"offset": 7797506048,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.6.attn_q.weight",
"offset": 7815331840,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.6.attn_v.weight",
"offset": 7833157632,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.7.attn_norm.weight",
"offset": 7837614080,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 7837630464,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 7900020736,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.7.ffn_up.weight",
"offset": 7962411008,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 8024801280,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 8024817664,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.7.attn_output.weight",
"offset": 8029274112,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.7.attn_q.weight",
"offset": 8047099904,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.7.attn_v.weight",
"offset": 8064925696,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.8.attn_norm.weight",
"offset": 8069382144,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 8069398528,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 8131788800,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.8.ffn_up.weight",
"offset": 8194179072,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 8256569344,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 8256585728,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.8.attn_output.weight",
"offset": 8261042176,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "63\u001e,+\u0001\u0003\ufffdI\fU\ufffd\ufffd\ufffdԦ\u0015\ufffd",
"offset": 8278867968,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.8.attn_v.weight",
"offset": 8296693760,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "blk.9.attn_norm.weight",
"offset": 8301150208,
"shape": [
4096
],
"size": 4456448,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 8301166592,
"shape": [
14336,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 8363556864,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.9.ffn_up.weight",
"offset": 8425947136,
"shape": [
4096,
14336
],
"size": 62390272,
"type": 8
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 8488337408,
"shape": [
4096
],
"size": 62390272,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 8488353792,
"shape": [
4096,
1024
],
"size": 16384,
"type": 8
},
{
"name": "blk.9.attn_output.weight",
"offset": 8492810240,
"shape": [
4096,
4096
],
"size": 4456448,
"type": 8
},
{
"name": "blk.9.attn_q.weight",
"offset": 8510636032,
"shape": [
4096,
4096
],
"size": 17825792,
"type": 8
},
{
"name": "blk.9.attn_v.weight",
"offset": 8528461824,
"shape": [
4096,
1024
],
"size": 17825792,
"type": 8
},
{
"name": "output_norm.weight",
"offset": 8532918272,
"shape": [
4096
],
"size": 4456448,
"type": 0
}
],
"version": 3
}