a small general purpose model with enticing roleplaying models. Introduce better RAG capabilities in the form of Bagel to Llama 3 8B Instruct as well as German multilanguage, higher general intelligence and vision support.
8B
399 Pulls Updated 7 weeks ago
744d54c5070d · 5.7GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 17,
"general.name": "Llama-3-8B-Ultra-Instruct",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"quantize.imatrix.chunks_count": 88,
"quantize.imatrix.dataset": "/training_data/groups_merged.txt",
"quantize.imatrix.entries_count": 224,
"quantize.imatrix.file": "/models/Llama-3-8B-Ultra-Instruct-GGUF/Llama-3-8B-Ultra-Instruct.imatrix",
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.pre": "llama-bpe",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "token_embd.weight",
"offset": 430940160,
"shape": [
4096,
128256
],
"size": 430940160,
"type": 13
},
{
"name": "blk.0.attn_norm.weight",
"offset": 792109056,
"shape": [
4096
],
"size": 361168896,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 792125440,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffdy\ufffd\u0000\u0000\u0000\u0000",
"offset": 840294400,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.0.ffn_up.weight",
"offset": 880664576,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 921034752,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 921051136,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.0.attn_output.weight",
"offset": 923934720,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.0.attn_q.weight",
"offset": 935469056,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.0.attn_v.weight",
"offset": 947003392,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 950444032,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 950460416,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 998629376,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.1.ffn_up.weight",
"offset": 1038999552,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 1079369728,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 1079386112,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.1.attn_output.weight",
"offset": 1082269696,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.1.attn_q.weight",
"offset": 1093804032,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.1.attn_v.weight",
"offset": 1105338368,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.10.attn_norm.weight",
"offset": 1108779008,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 1108795392,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 1156964352,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.10.ffn_up.weight",
"offset": 1197334528,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 1237704704,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 1237721088,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.10.attn_output.weight",
"offset": 1240604672,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.10.attn_q.weight",
"offset": 1252139008,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.10.attn_v.weight",
"offset": 1263673344,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 1267113984,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 1267130368,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 1315299328,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.11.ffn_up.weight",
"offset": 1355669504,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 1396039680,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 1396056064,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.11.attn_output.weight",
"offset": 1398939648,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.11.attn_q.weight",
"offset": 1410473984,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.11.attn_v.weight",
"offset": 1422008320,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 1425448960,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 1425465344,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 1465835520,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.12.ffn_up.weight",
"offset": 1506205696,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 1546575872,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 1546592256,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.12.attn_output.weight",
"offset": 1549475840,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.12.attn_q.weight",
"offset": 1561010176,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.12.attn_v.weight",
"offset": 1572544512,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.13.attn_norm.weight",
"offset": 1575428096,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 1575444480,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 1615814656,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.13.ffn_up.weight",
"offset": 1656184832,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 1696555008,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 1696571392,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.13.attn_output.weight",
"offset": 1699454976,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.13.attn_q.weight",
"offset": 1710989312,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.13.attn_v.weight",
"offset": 1722523648,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.14.attn_norm.weight",
"offset": 1725407232,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 1725423616,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 1773592576,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.14.ffn_up.weight",
"offset": 1813962752,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 1854332928,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 1854349312,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.14.attn_output.weight",
"offset": 1857232896,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.14.attn_q.weight",
"offset": 1868767232,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.14.attn_v.weight",
"offset": 1880301568,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 1883742208,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 1883758592,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 1924128768,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.15.ffn_up.weight",
"offset": 1964498944,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 2004869120,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 2004885504,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.15.attn_output.weight",
"offset": 2007769088,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.15.attn_q.weight",
"offset": 2019303424,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.15.attn_v.weight",
"offset": 2030837760,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.16.attn_norm.weight",
"offset": 2033721344,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 2033737728,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 2074107904,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.16.ffn_up.weight",
"offset": 2114478080,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 2154848256,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 2154864640,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.16.attn_output.weight",
"offset": 2157748224,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.16.attn_q.weight",
"offset": 2169282560,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.16.attn_v.weight",
"offset": 2180816896,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.17.attn_norm.weight",
"offset": 2183700480,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 2183716864,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 2231885824,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.17.ffn_up.weight",
"offset": 2272256000,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 2312626176,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 2312642560,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.17.attn_output.weight",
"offset": 2315526144,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.17.attn_q.weight",
"offset": 2327060480,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.17.attn_v.weight",
"offset": 2338594816,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 2342035456,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 2342051840,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 2382422016,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.18.ffn_up.weight",
"offset": 2422792192,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 2463162368,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 2463178752,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.18.attn_output.weight",
"offset": 2466062336,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.18.attn_q.weight",
"offset": 2477596672,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.18.attn_v.weight",
"offset": 2489131008,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.19.attn_norm.weight",
"offset": 2492014592,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 2492030976,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 2532401152,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.19.ffn_up.weight",
"offset": 2572771328,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 2613141504,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 2613157888,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.19.attn_output.weight",
"offset": 2616041472,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.19.attn_q.weight",
"offset": 2627575808,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.19.attn_v.weight",
"offset": 2639110144,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.2.attn_norm.weight",
"offset": 2641993728,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 2642010112,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 2690179072,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.2.ffn_up.weight",
"offset": 2730549248,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 2770919424,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 2770935808,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.2.attn_output.weight",
"offset": 2773819392,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.2.attn_q.weight",
"offset": 2785353728,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.2.attn_v.weight",
"offset": 2796888064,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 2800328704,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 2800345088,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 2840715264,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.20.ffn_up.weight",
"offset": 2881085440,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 2921455616,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 2921472000,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.20.attn_output.weight",
"offset": 2924355584,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.20.attn_q.weight",
"offset": 2935889920,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.20.attn_v.weight",
"offset": 2947424256,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.21.attn_norm.weight",
"offset": 2950307840,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 2950324224,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 2990694400,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.21.ffn_up.weight",
"offset": 3031064576,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 3071434752,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 3071451136,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.21.attn_output.weight",
"offset": 3074334720,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.21.attn_q.weight",
"offset": 3085869056,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.21.attn_v.weight",
"offset": 3097403392,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.22.attn_norm.weight",
"offset": 3100286976,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 3100303360,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 3148472320,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.22.ffn_up.weight",
"offset": 3188842496,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 3229212672,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 3229229056,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.22.attn_output.weight",
"offset": 3232112640,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.22.attn_q.weight",
"offset": 3243646976,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.22.attn_v.weight",
"offset": 3255181312,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 3258621952,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 3258638336,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 3299008512,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.23.ffn_up.weight",
"offset": 3339378688,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 3379748864,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 3379765248,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.23.attn_output.weight",
"offset": 3382648832,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.23.attn_q.weight",
"offset": 3394183168,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.23.attn_v.weight",
"offset": 3405717504,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.24.attn_norm.weight",
"offset": 3408601088,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 3408617472,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 3448987648,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.24.ffn_up.weight",
"offset": 3489357824,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 3529728000,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 3529744384,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.24.attn_output.weight",
"offset": 3532627968,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.24.attn_q.weight",
"offset": 3544162304,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.24.attn_v.weight",
"offset": 3555696640,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.25.attn_norm.weight",
"offset": 3558580224,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 3558596608,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 3606765568,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.25.ffn_up.weight",
"offset": 3647135744,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 3687505920,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 3687522304,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.25.attn_output.weight",
"offset": 3690405888,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.25.attn_q.weight",
"offset": 3701940224,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.25.attn_v.weight",
"offset": 3713474560,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.26.attn_norm.weight",
"offset": 3716915200,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 3716931584,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 3757301760,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.26.ffn_up.weight",
"offset": 3797671936,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 3838042112,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 3838058496,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.26.attn_output.weight",
"offset": 3840942080,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.26.attn_q.weight",
"offset": 3852476416,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.26.attn_v.weight",
"offset": 3864010752,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.27.attn_norm.weight",
"offset": 3866894336,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 3866910720,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 3907280896,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.27.ffn_up.weight",
"offset": 3947651072,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 3988021248,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 3988037632,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.27.attn_output.weight",
"offset": 3990921216,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.27.attn_q.weight",
"offset": 4002455552,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.27.attn_v.weight",
"offset": 4013989888,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.28.attn_norm.weight",
"offset": 4016873472,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 4016889856,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 4065058816,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.28.ffn_up.weight",
"offset": 4105428992,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 4145799168,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 4145815552,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.28.attn_output.weight",
"offset": 4148699136,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.28.attn_q.weight",
"offset": 4160233472,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.28.attn_v.weight",
"offset": 4171767808,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 4175208448,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 4175224832,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 4215595008,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.29.ffn_up.weight",
"offset": 4255965184,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 4296335360,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 4296351744,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.29.attn_output.weight",
"offset": 4299235328,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.29.attn_q.weight",
"offset": 4310769664,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.29.attn_v.weight",
"offset": 4322304000,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.3.attn_norm.weight",
"offset": 4325187584,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 4325203968,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 4365574144,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.3.ffn_up.weight",
"offset": 4405944320,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 4446314496,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 4446330880,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.3.attn_output.weight",
"offset": 4449214464,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.3.attn_q.weight",
"offset": 4460748800,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.3.attn_v.weight",
"offset": 4472283136,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.30.attn_norm.weight",
"offset": 4475166720,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 4475183104,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 4523352064,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.30.ffn_up.weight",
"offset": 4563722240,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 4604092416,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 4604108800,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.30.attn_output.weight",
"offset": 4606992384,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.30.attn_q.weight",
"offset": 4618526720,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.30.attn_v.weight",
"offset": 4630061056,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 4633501696,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 4633518080,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 4673888256,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.31.ffn_up.weight",
"offset": 4714258432,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 4754628608,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 4754644992,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.31.attn_output.weight",
"offset": 4757528576,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.31.attn_q.weight",
"offset": 4769062912,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.31.attn_v.weight",
"offset": 4780597248,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.4.attn_norm.weight",
"offset": 4783480832,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 4783497216,
"shape": [
14336,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 4823867392,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.4.ffn_up.weight",
"offset": 4864237568,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 4904607744,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 4904624128,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.4.attn_output.weight",
"offset": 4907507712,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.4.attn_q.weight",
"offset": 4919042048,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.4.attn_v.weight",
"offset": 4930576384,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 13
},
{
"name": "blk.5.attn_norm.weight",
"offset": 4933459968,
"shape": [
4096
],
"size": 2883584,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 4933476352,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 4981645312,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.5.ffn_up.weight",
"offset": 5022015488,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 5062385664,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 5062402048,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.5.attn_output.weight",
"offset": 5065285632,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.5.attn_q.weight",
"offset": 5076819968,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.5.attn_v.weight",
"offset": 5088354304,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 5091794944,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 5091811328,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 5139980288,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.6.ffn_up.weight",
"offset": 5180350464,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 5220720640,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 5220737024,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.6.attn_output.weight",
"offset": 5223620608,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.6.attn_q.weight",
"offset": 5235154944,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.6.attn_v.weight",
"offset": 5246689280,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 5250129920,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 5250146304,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 5298315264,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.7.ffn_up.weight",
"offset": 5338685440,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 5379055616,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 5379072000,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.7.attn_output.weight",
"offset": 5381955584,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.7.attn_q.weight",
"offset": 5393489920,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.7.attn_v.weight",
"offset": 5405024256,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 5408464896,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 5408481280,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 5456650240,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.8.ffn_up.weight",
"offset": 5497020416,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 5537390592,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 5537406976,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.8.attn_output.weight",
"offset": 5540290560,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "\ufffd\ufffd\u0004\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdi\ufffd\u001a\ufffd\ufffdb\u0019^\ufffd\ufffd",
"offset": 5551824896,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.8.attn_v.weight",
"offset": 5563359232,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 5566799872,
"shape": [
4096
],
"size": 3440640,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 5566816256,
"shape": [
14336,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 5614985216,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 13
},
{
"name": "blk.9.ffn_up.weight",
"offset": 5655355392,
"shape": [
4096,
14336
],
"size": 40370176,
"type": 13
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 5695725568,
"shape": [
4096
],
"size": 40370176,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 5695741952,
"shape": [
4096,
1024
],
"size": 16384,
"type": 13
},
{
"name": "blk.9.attn_output.weight",
"offset": 5698625536,
"shape": [
4096,
4096
],
"size": 2883584,
"type": 13
},
{
"name": "blk.9.attn_q.weight",
"offset": 5710159872,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.9.attn_v.weight",
"offset": 5721694208,
"shape": [
4096,
1024
],
"size": 11534336,
"type": 14
},
{
"name": "output_norm.weight",
"offset": 5725134848,
"shape": [
4096
],
"size": 3440640,
"type": 0
}
],
"version": 3
}