a small general purpose model with enticing roleplaying models. Introduce better RAG capabilities in the form of Bagel to Llama 3 8B Instruct as well as German multilanguage, higher general intelligence and vision support.
8B
400 Pulls Updated 7 weeks ago
9de14a314423 · 4.9GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 15,
"general.name": "Llama-3-8B-Ultra-Instruct",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"quantize.imatrix.chunks_count": 88,
"quantize.imatrix.dataset": "/training_data/groups_merged.txt",
"quantize.imatrix.entries_count": 224,
"quantize.imatrix.file": "/models/Llama-3-8B-Ultra-Instruct-GGUF/Llama-3-8B-Ultra-Instruct.imatrix",
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.pre": "llama-bpe",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "token_embd.weight",
"offset": 430940160,
"shape": [
4096,
128256
],
"size": 430940160,
"type": 12
},
{
"name": "blk.0.attn_norm.weight",
"offset": 726441984,
"shape": [
4096
],
"size": 295501824,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 726458368,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 774627328,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.0.ffn_up.weight",
"offset": 807657472,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 840687616,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 840704000,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.0.attn_output.weight",
"offset": 843063296,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.0.attn_q.weight",
"offset": 852500480,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.0.attn_v.weight",
"offset": 861937664,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 865378304,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 865394688,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 913563648,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.1.ffn_up.weight",
"offset": 946593792,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 979623936,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 979640320,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.1.attn_output.weight",
"offset": 981999616,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.1.attn_q.weight",
"offset": 991436800,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.1.attn_v.weight",
"offset": 1000873984,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.10.attn_norm.weight",
"offset": 1004314624,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 1004331008,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 1052499968,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.10.ffn_up.weight",
"offset": 1085530112,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 1118560256,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 1118576640,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.10.attn_output.weight",
"offset": 1120935936,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.10.attn_q.weight",
"offset": 1130373120,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.10.attn_v.weight",
"offset": 1139810304,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 1143250944,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 1143267328,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 1191436288,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.11.ffn_up.weight",
"offset": 1224466432,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 1257496576,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 1257512960,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.11.attn_output.weight",
"offset": 1259872256,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.11.attn_q.weight",
"offset": 1269309440,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.11.attn_v.weight",
"offset": 1278746624,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 1282187264,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 1282203648,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 1315233792,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.12.ffn_up.weight",
"offset": 1348263936,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 1381294080,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 1381310464,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.12.attn_output.weight",
"offset": 1383669760,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.12.attn_q.weight",
"offset": 1393106944,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.12.attn_v.weight",
"offset": 1402544128,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.13.attn_norm.weight",
"offset": 1404903424,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 1404919808,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 1437949952,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.13.ffn_up.weight",
"offset": 1470980096,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 1504010240,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 1504026624,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.13.attn_output.weight",
"offset": 1506385920,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.13.attn_q.weight",
"offset": 1515823104,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.13.attn_v.weight",
"offset": 1525260288,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.14.attn_norm.weight",
"offset": 1527619584,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 1527635968,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 1575804928,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.14.ffn_up.weight",
"offset": 1608835072,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 1641865216,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 1641881600,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.14.attn_output.weight",
"offset": 1644240896,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.14.attn_q.weight",
"offset": 1653678080,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.14.attn_v.weight",
"offset": 1663115264,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 1666555904,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 1666572288,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 1699602432,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.15.ffn_up.weight",
"offset": 1732632576,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 1765662720,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 1765679104,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.15.attn_output.weight",
"offset": 1768038400,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.15.attn_q.weight",
"offset": 1777475584,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.15.attn_v.weight",
"offset": 1786912768,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "weight\u0002\u0000\u0000\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\u00008\u0000\u0000\u0000",
"offset": 1789272064,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 1789288448,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 1822318592,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.16.ffn_up.weight",
"offset": 1855348736,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 1888378880,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 1888395264,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.16.attn_output.weight",
"offset": 1890754560,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.16.attn_q.weight",
"offset": 1900191744,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.16.attn_v.weight",
"offset": 1909628928,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.17.attn_norm.weight",
"offset": 1911988224,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 1912004608,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 1960173568,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.17.ffn_up.weight",
"offset": 1993203712,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 2026233856,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 2026250240,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.17.attn_output.weight",
"offset": 2028609536,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.17.attn_q.weight",
"offset": 2038046720,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.17.attn_v.weight",
"offset": 2047483904,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 2050924544,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 2050940928,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 2083971072,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.18.ffn_up.weight",
"offset": 2117001216,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 2150031360,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 2150047744,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.18.attn_output.weight",
"offset": 2152407040,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.18.attn_q.weight",
"offset": 2161844224,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.18.attn_v.weight",
"offset": 2171281408,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.19.attn_norm.weight",
"offset": 2173640704,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 2173657088,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 2206687232,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.19.ffn_up.weight",
"offset": 2239717376,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 2272747520,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 2272763904,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.19.attn_output.weight",
"offset": 2275123200,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.19.attn_q.weight",
"offset": 2284560384,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.19.attn_v.weight",
"offset": 2293997568,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.2.attn_norm.weight",
"offset": 2296356864,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 2296373248,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 2344542208,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.2.ffn_up.weight",
"offset": 2377572352,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 2410602496,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 2410618880,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.2.attn_output.weight",
"offset": 2412978176,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.2.attn_q.weight",
"offset": 2422415360,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.2.attn_v.weight",
"offset": 2431852544,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 2435293184,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 2435309568,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 2468339712,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.20.ffn_up.weight",
"offset": 2501369856,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 2534400000,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 2534416384,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.20.attn_output.weight",
"offset": 2536775680,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.20.attn_q.weight",
"offset": 2546212864,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.20.attn_v.weight",
"offset": 2555650048,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.21.attn_norm.weight",
"offset": 2558009344,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 2558025728,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 2591055872,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.21.ffn_up.weight",
"offset": 2624086016,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 2657116160,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 2657132544,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.21.attn_output.weight",
"offset": 2659491840,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.21.attn_q.weight",
"offset": 2668929024,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.21.attn_v.weight",
"offset": 2678366208,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.22.attn_norm.weight",
"offset": 2680725504,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 2680741888,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 2728910848,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.22.ffn_up.weight",
"offset": 2761940992,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 2794971136,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 2794987520,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.22.attn_output.weight",
"offset": 2797346816,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.22.attn_q.weight",
"offset": 2806784000,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.22.attn_v.weight",
"offset": 2816221184,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 2819661824,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 2819678208,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 2852708352,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.23.ffn_up.weight",
"offset": 2885738496,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 2918768640,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 2918785024,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.23.attn_output.weight",
"offset": 2921144320,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.23.attn_q.weight",
"offset": 2930581504,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.23.attn_v.weight",
"offset": 2940018688,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.24.attn_norm.weight",
"offset": 2942377984,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 2942394368,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 2975424512,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.24.ffn_up.weight",
"offset": 3008454656,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 3041484800,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 3041501184,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.24.attn_output.weight",
"offset": 3043860480,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.24.attn_q.weight",
"offset": 3053297664,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.24.attn_v.weight",
"offset": 3062734848,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.25.attn_norm.weight",
"offset": 3065094144,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 3065110528,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 3113279488,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.25.ffn_up.weight",
"offset": 3146309632,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 3179339776,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 3179356160,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.25.attn_output.weight",
"offset": 3181715456,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.25.attn_q.weight",
"offset": 3191152640,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.25.attn_v.weight",
"offset": 3200589824,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.26.attn_norm.weight",
"offset": 3204030464,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 3204046848,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 3237076992,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.26.ffn_up.weight",
"offset": 3270107136,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 3303137280,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 3303153664,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.26.attn_output.weight",
"offset": 3305512960,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.26.attn_q.weight",
"offset": 3314950144,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.26.attn_v.weight",
"offset": 3324387328,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.27.attn_norm.weight",
"offset": 3326746624,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 3326763008,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 3359793152,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.27.ffn_up.weight",
"offset": 3392823296,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 3425853440,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 3425869824,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.27.attn_output.weight",
"offset": 3428229120,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.27.attn_q.weight",
"offset": 3437666304,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.27.attn_v.weight",
"offset": 3447103488,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.28.attn_norm.weight",
"offset": 3449462784,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 3449479168,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 3497648128,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.28.ffn_up.weight",
"offset": 3530678272,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 3563708416,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 3563724800,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.28.attn_output.weight",
"offset": 3566084096,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.28.attn_q.weight",
"offset": 3575521280,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.28.attn_v.weight",
"offset": 3584958464,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 3588399104,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 3588415488,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 3621445632,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.29.ffn_up.weight",
"offset": 3654475776,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 3687505920,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 3687522304,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.29.attn_output.weight",
"offset": 3689881600,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.29.attn_q.weight",
"offset": 3699318784,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.29.attn_v.weight",
"offset": 3708755968,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.3.attn_norm.weight",
"offset": 3711115264,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 3711131648,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 3744161792,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.3.ffn_up.weight",
"offset": 3777191936,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 3810222080,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 3810238464,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.3.attn_output.weight",
"offset": 3812597760,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.3.attn_q.weight",
"offset": 3822034944,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.3.attn_v.weight",
"offset": 3831472128,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.30.attn_norm.weight",
"offset": 3833831424,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 3833847808,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 3882016768,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.30.ffn_up.weight",
"offset": 3915046912,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 3948077056,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 3948093440,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.30.attn_output.weight",
"offset": 3950452736,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.30.attn_q.weight",
"offset": 3959889920,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.30.attn_v.weight",
"offset": 3969327104,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 3972767744,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 3972784128,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 4005814272,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.31.ffn_up.weight",
"offset": 4038844416,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 4071874560,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 4071890944,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.31.attn_output.weight",
"offset": 4074250240,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.31.attn_q.weight",
"offset": 4083687424,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.31.attn_v.weight",
"offset": 4093124608,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.4.attn_norm.weight",
"offset": 4095483904,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 4095500288,
"shape": [
14336,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 4128530432,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.4.ffn_up.weight",
"offset": 4161560576,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 4194590720,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 4194607104,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.4.attn_output.weight",
"offset": 4196966400,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.4.attn_q.weight",
"offset": 4206403584,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.4.attn_v.weight",
"offset": 4215840768,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 12
},
{
"name": "blk.5.attn_norm.weight",
"offset": 4218200064,
"shape": [
4096
],
"size": 2359296,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 4218216448,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 4266385408,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.5.ffn_up.weight",
"offset": 4299415552,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 4332445696,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 4332462080,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.5.attn_output.weight",
"offset": 4334821376,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.5.attn_q.weight",
"offset": 4344258560,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.5.attn_v.weight",
"offset": 4353695744,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 4357136384,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 4357152768,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 4405321728,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.6.ffn_up.weight",
"offset": 4438351872,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 4471382016,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 4471398400,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.6.attn_output.weight",
"offset": 4473757696,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.6.attn_q.weight",
"offset": 4483194880,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.6.attn_v.weight",
"offset": 4492632064,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 4496072704,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 4496089088,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 4544258048,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.7.ffn_up.weight",
"offset": 4577288192,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 4610318336,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 4610334720,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.7.attn_output.weight",
"offset": 4612694016,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.7.attn_q.weight",
"offset": 4622131200,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.7.attn_v.weight",
"offset": 4631568384,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 4635009024,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 4635025408,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 4683194368,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.8.ffn_up.weight",
"offset": 4716224512,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 4749254656,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 4749271040,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.8.attn_output.weight",
"offset": 4751630336,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.8.attn_q.weight",
"offset": 4761067520,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.8.attn_v.weight",
"offset": 4770504704,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 4773945344,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 4773961728,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 4822130688,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 12
},
{
"name": "blk.9.ffn_up.weight",
"offset": 4855160832,
"shape": [
4096,
14336
],
"size": 33030144,
"type": 12
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 4888190976,
"shape": [
4096
],
"size": 33030144,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 4888207360,
"shape": [
4096,
1024
],
"size": 16384,
"type": 12
},
{
"name": "blk.9.attn_output.weight",
"offset": 4890566656,
"shape": [
4096,
4096
],
"size": 2359296,
"type": 12
},
{
"name": "blk.9.attn_q.weight",
"offset": 4900003840,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.9.attn_v.weight",
"offset": 4909441024,
"shape": [
4096,
1024
],
"size": 9437184,
"type": 14
},
{
"name": "output_norm.weight",
"offset": 4912881664,
"shape": [
4096
],
"size": 3440640,
"type": 0
}
],
"version": 3
}