qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 17, "general.name": "Qwen2-72B", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.00001, "qwen2.block_count": 80, "qwen2.context_length": 131072, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 856424448, "shape": [ 8192, 29568 ], "size": 856424448, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 1022951424, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.0.attn_k.bias", "offset": 1189478400, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 1189482496, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 1195249664, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.0.attn_q.bias", "offset": 1241387008, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 1241419776, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.0.attn_v.bias", "offset": 1287557120, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 1287561216, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.0.attn_norm.weight", "offset": 1294442496, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 1294475264, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 1551835136, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1551867904, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1551900672, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.1.ffn_gate.weight", "offset": 1809260544, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 1975787520, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 2142314496, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 2142347264, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 2142351360, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 2148118528, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.1.attn_q.bias", "offset": 2194255872, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 2194288640, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.1.attn_v.bias", "offset": 2240425984, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 2240430080, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 2247311360, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 2247344128, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.2.ffn_gate.weight", "offset": 2504704000, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 2671230976, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 2837757952, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 2837790720, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 2837794816, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 2843561984, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.2.attn_q.bias", "offset": 2889699328, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 2889732096, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.2.attn_v.bias", "offset": 2935869440, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 2935873536, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 2942754816, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 2942787584, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.3.ffn_gate.weight", "offset": 3200147456, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 3366674432, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 3533201408, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 3533234176, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 3533238272, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 3539005440, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.3.attn_q.bias", "offset": 3585142784, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 3585175552, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.3.attn_v.bias", "offset": 3631312896, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 3631316992, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 3638198272, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 3638231040, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.4.ffn_gate.weight", "offset": 3895590912, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 4062117888, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 4228644864, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 4228677632, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 4228681728, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 4234448896, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.4.attn_q.bias", "offset": 4280586240, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 4280619008, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.4.attn_v.bias", "offset": 4326756352, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 4326760448, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.5.attn_k.bias", "offset": 4333641728, "shape": [ 1024 ], "size": 6881280, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 4333645824, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 4339412992, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.5.attn_q.bias", "offset": 4385550336, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 4385583104, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.5.attn_v.bias", "offset": 4431720448, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 4431724544, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 4438605824, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 4438638592, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.5.ffn_gate.weight", "offset": 4695998464, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 4862525440, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 5029052416, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 5029085184, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 5029117952, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.6.ffn_gate.weight", "offset": 5286477824, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 5453004800, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 5619531776, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 5619564544, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 5619568640, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 5625335808, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.6.attn_q.bias", "offset": 5671473152, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 5671505920, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.6.attn_v.bias", "offset": 5717643264, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 5717647360, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 5724528640, "shape": [ 8192, 29568 ], "size": 6881280, "type": 13 }, { "name": "blk.7.attn_k.bias", "offset": 5891055616, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 5891059712, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 5896826880, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.7.attn_q.bias", "offset": 5942964224, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 5942996992, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.7.attn_v.bias", "offset": 5989134336, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 5989138432, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 5996019712, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 5996052480, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 6253412352, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 6419939328, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 6419972096, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 6420004864, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.8.ffn_gate.weight", "offset": 6677364736, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 6843891712, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 7010418688, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 7010451456, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 7010455552, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 7016222720, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.8.attn_q.bias", "offset": 7062360064, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 7062392832, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.8.attn_v.bias", "offset": 7108530176, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 7108534272, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 7115415552, "shape": [ 8192, 29568 ], "size": 6881280, "type": 13 }, { "name": "blk.9.ffn_up.weight", "offset": 7281942528, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.9.attn_k.bias", "offset": 7448469504, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 7448473600, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 7454240768, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.9.attn_q.bias", "offset": 7500378112, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 7500410880, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.9.attn_v.bias", "offset": 7546548224, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 7546552320, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 7553433600, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 7553466368, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.10.ffn_gate.weight", "offset": 7810826240, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.10.ffn_up.weight", "offset": 7977353216, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 8143880192, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 8143912960, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 8143917056, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 8149684224, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.10.attn_q.bias", "offset": 8195821568, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 8195854336, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.10.attn_v.bias", "offset": 8241991680, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 8241995776, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 8247762944, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 8247795712, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.11.ffn_gate.weight", "offset": 8429461504, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.11.ffn_up.weight", "offset": 8595988480, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 8762515456, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 8762548224, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 8762552320, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 8768319488, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.11.attn_q.bias", "offset": 8814456832, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 8814489600, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.11.attn_v.bias", "offset": 8860626944, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 8860631040, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 8866398208, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 8866430976, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 9048096768, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 9048129536, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 9048162304, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.12.ffn_gate.weight", "offset": 9305522176, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.12.ffn_up.weight", "offset": 9472049152, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 9638576128, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 9638608896, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 9638612992, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 9644380160, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.12.attn_q.bias", "offset": 9690517504, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 9690550272, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.12.attn_v.bias", "offset": 9736687616, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 9736691712, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 9743572992, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 9743605760, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.13.ffn_gate.weight", "offset": 9925271552, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.13.ffn_up.weight", "offset": 10091798528, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 10258325504, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 10258358272, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 10258362368, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 10264129536, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.13.attn_q.bias", "offset": 10310266880, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 10310299648, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.13.attn_v.bias", "offset": 10356436992, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 10356441088, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_k.bias", "offset": 10362208256, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 10362212352, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 10367979520, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.14.attn_q.bias", "offset": 10414116864, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 10414149632, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.14.attn_v.bias", "offset": 10460286976, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 10460291072, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 10466058240, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 10466091008, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.14.ffn_gate.weight", "offset": 10647756800, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 10814283776, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 10980810752, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 10980843520, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 10980876288, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.15.ffn_gate.weight", "offset": 11238236160, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 11404763136, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 11571290112, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 11571322880, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 11571326976, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 11577094144, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.15.attn_q.bias", "offset": 11623231488, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 11623264256, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.15.attn_v.bias", "offset": 11669401600, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 11669405696, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 11676286976, "shape": [ 8192, 29568 ], "size": 6881280, "type": 13 }, { "name": "blk.16.attn_k.bias", "offset": 11842813952, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 11842818048, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 11848585216, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.16.attn_q.bias", "offset": 11894722560, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 11894755328, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.16.attn_v.bias", "offset": 11940892672, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 11940896768, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 11946663936, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 11946696704, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 12128362496, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 12294889472, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 12294922240, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 12294955008, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.17.ffn_gate.weight", "offset": 12476620800, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.17.ffn_up.weight", "offset": 12643147776, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 12809674752, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 12809707520, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 12809711616, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 12815478784, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.17.attn_q.bias", "offset": 12861616128, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 12861648896, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.17.attn_v.bias", "offset": 12907786240, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 12907790336, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 12913557504, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 13080084480, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.18.attn_k.bias", "offset": 13246611456, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 13246615552, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 13252382720, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.18.attn_q.bias", "offset": 13298520064, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 13298552832, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.18.attn_v.bias", "offset": 13344690176, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 13344694272, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 13351575552, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 13351608320, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 13608968192, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 13609000960, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 13609033728, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.19.ffn_gate.weight", "offset": 13790699520, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.19.ffn_up.weight", "offset": 13957226496, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 14123753472, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 14123786240, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 14123790336, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 14129557504, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.19.attn_q.bias", "offset": 14175694848, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 14175727616, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.19.attn_v.bias", "offset": 14221864960, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 14221869056, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 14227636224, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 14227668992, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.20.ffn_gate.weight", "offset": 14409334784, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 14575861760, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 14742388736, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 14742421504, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 14742425600, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 14748192768, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.20.attn_q.bias", "offset": 14794330112, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 14794362880, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.20.attn_v.bias", "offset": 14840500224, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 14840504320, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 14846271488, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 14846304256, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.21.ffn_gate.weight", "offset": 15103664128, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 15270191104, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 15436718080, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 15436750848, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 15436754944, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 15442522112, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.21.attn_q.bias", "offset": 15488659456, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 15488692224, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.21.attn_v.bias", "offset": 15534829568, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 15534833664, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 15541714944, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 15541747712, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.22.ffn_gate.weight", "offset": 15723413504, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.22.ffn_up.weight", "offset": 15889940480, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 16056467456, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 16056500224, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 16056504320, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 16062271488, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.22.attn_q.bias", "offset": 16108408832, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 16108441600, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.22.attn_v.bias", "offset": 16154578944, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 16154583040, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_k.bias", "offset": 16160350208, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 16160354304, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 16166121472, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.23.attn_q.bias", "offset": 16212258816, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 16212291584, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.23.attn_v.bias", "offset": 16258428928, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 16258433024, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 16264200192, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 16264232960, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.23.ffn_gate.weight", "offset": 16445898752, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 16612425728, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 16778952704, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 16778985472, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 16779018240, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.24.ffn_gate.weight", "offset": 17036378112, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 17202905088, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 17369432064, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 17369464832, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 17369468928, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 17375236096, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.24.attn_q.bias", "offset": 17421373440, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 17421406208, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.24.attn_v.bias", "offset": 17467543552, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 17467547648, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 17474428928, "shape": [ 8192, 29568 ], "size": 6881280, "type": 13 }, { "name": "blk.25.attn_k.bias", "offset": 17640955904, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 17640960000, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 17646727168, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.25.attn_q.bias", "offset": 17692864512, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 17692897280, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.25.attn_v.bias", "offset": 17739034624, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 17739038720, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 17744805888, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 17744838656, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.25.ffn_up.weight", "offset": 17926504448, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 18093031424, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 18093064192, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 18093096960, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.26.ffn_gate.weight", "offset": 18274762752, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 18441289728, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 18607816704, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 18607849472, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 18607853568, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 18613620736, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.26.attn_q.bias", "offset": 18659758080, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 18659790848, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.26.attn_v.bias", "offset": 18705928192, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 18705932288, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 18711699456, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 18878226432, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.27.attn_k.bias", "offset": 19044753408, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 19044757504, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.attn_output.weight", "offset": 19050524672, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.27.attn_q.bias", "offset": 19096662016, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 19096694784, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.27.attn_v.bias", "offset": 19142832128, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 19142836224, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 19149717504, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 19149750272, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.27.ffn_norm.weight", "offset": 19407110144, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 19407142912, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 19407175680, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.28.ffn_gate.weight", "offset": 19588841472, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.28.ffn_up.weight", "offset": 19755368448, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.28.ffn_norm.weight", "offset": 19921895424, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 19921928192, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 19921932288, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.28.attn_output.weight", "offset": 19927699456, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.28.attn_q.bias", "offset": 19973836800, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 19973869568, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.28.attn_v.bias", "offset": 20020006912, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 20020011008, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 20025778176, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 20025810944, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.29.ffn_gate.weight", "offset": 20207476736, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.29.ffn_up.weight", "offset": 20374003712, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.29.ffn_norm.weight", "offset": 20540530688, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 20540563456, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 20540567552, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.29.attn_output.weight", "offset": 20546334720, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.29.attn_q.bias", "offset": 20592472064, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 20592504832, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.29.attn_v.bias", "offset": 20638642176, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 20638646272, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 20644413440, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 20644446208, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.30.ffn_gate.weight", "offset": 20901806080, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.30.ffn_up.weight", "offset": 21068333056, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.30.ffn_norm.weight", "offset": 21234860032, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 21234892800, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 21234896896, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.30.attn_output.weight", "offset": 21240664064, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.30.attn_q.bias", "offset": 21286801408, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 21286834176, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.30.attn_v.bias", "offset": 21332971520, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 21332975616, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.31.attn_norm.weight", "offset": 21339856896, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 21339889664, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.31.ffn_gate.weight", "offset": 21521555456, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.31.ffn_up.weight", "offset": 21688082432, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.31.ffn_norm.weight", "offset": 21854609408, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 21854642176, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 21854646272, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.31.attn_output.weight", "offset": 21860413440, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.31.attn_q.bias", "offset": 21906550784, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 21906583552, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.31.attn_v.bias", "offset": 21952720896, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 21952724992, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_k.bias", "offset": 21958492160, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 21958496256, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_output.weight", "offset": 21964263424, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.32.attn_q.bias", "offset": 22010400768, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 22010433536, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.32.attn_v.bias", "offset": 22056570880, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 22056574976, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 22062342144, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 22062374912, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.32.ffn_gate.weight", "offset": 22244040704, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.32.ffn_up.weight", "offset": 22410567680, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.32.ffn_norm.weight", "offset": 22577094656, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 22577127424, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 22577160192, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.33.ffn_gate.weight", "offset": 22834520064, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.33.ffn_up.weight", "offset": 23001047040, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.33.ffn_norm.weight", "offset": 23167574016, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 23167606784, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 23167610880, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.33.attn_output.weight", "offset": 23173378048, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.33.attn_q.bias", "offset": 23219515392, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 23219548160, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.33.attn_v.bias", "offset": 23265685504, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 23265689600, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.34.ffn_gate.weight", "offset": 23272570880, "shape": [ 8192, 29568 ], "size": 6881280, "type": 13 }, { "name": "blk.34.attn_k.bias", "offset": 23439097856, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 23439101952, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.attn_output.weight", "offset": 23444869120, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.34.attn_q.bias", "offset": 23491006464, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 23491039232, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.34.attn_v.bias", "offset": 23537176576, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 23537180672, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 23542947840, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 23542980608, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.34.ffn_up.weight", "offset": 23724646400, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.34.ffn_norm.weight", "offset": 23891173376, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 23891206144, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 23891238912, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.35.ffn_gate.weight", "offset": 24072904704, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.35.ffn_up.weight", "offset": 24239431680, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.35.ffn_norm.weight", "offset": 24405958656, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 24405991424, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 24405995520, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.35.attn_output.weight", "offset": 24411762688, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.35.attn_q.bias", "offset": 24457900032, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 24457932800, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.35.attn_v.bias", "offset": 24504070144, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 24504074240, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 24509841408, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.36.ffn_up.weight", "offset": 24676368384, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.36.attn_k.bias", "offset": 24842895360, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 24842899456, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.attn_output.weight", "offset": 24848666624, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.36.attn_q.bias", "offset": 24894803968, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 24894836736, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.36.attn_v.bias", "offset": 24940974080, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 24940978176, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.36.attn_norm.weight", "offset": 24947859456, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 24947892224, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.36.ffn_norm.weight", "offset": 25205252096, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 25205284864, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 25205317632, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.37.ffn_gate.weight", "offset": 25386983424, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.37.ffn_up.weight", "offset": 25553510400, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.37.ffn_norm.weight", "offset": 25720037376, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 25720070144, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 25720074240, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.37.attn_output.weight", "offset": 25725841408, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.37.attn_q.bias", "offset": 25771978752, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 25772011520, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.37.attn_v.bias", "offset": 25818148864, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 25818152960, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 25823920128, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 25823952896, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.38.ffn_gate.weight", "offset": 26005618688, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.38.ffn_up.weight", "offset": 26172145664, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.38.ffn_norm.weight", "offset": 26338672640, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 26338705408, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 26338709504, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.38.attn_output.weight", "offset": 26344476672, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.38.attn_q.bias", "offset": 26390614016, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 26390646784, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.38.attn_v.bias", "offset": 26436784128, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 26436788224, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 26442555392, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 26442588160, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.39.ffn_gate.weight", "offset": 26699948032, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.39.ffn_up.weight", "offset": 26866475008, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.39.ffn_norm.weight", "offset": 27033001984, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 27033034752, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 27033038848, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.39.attn_output.weight", "offset": 27038806016, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.39.attn_q.bias", "offset": 27084943360, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 27084976128, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.39.attn_v.bias", "offset": 27131113472, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 27131117568, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.40.attn_norm.weight", "offset": 27137998848, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 27138031616, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.40.ffn_gate.weight", "offset": 27319697408, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.40.ffn_up.weight", "offset": 27486224384, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.40.ffn_norm.weight", "offset": 27652751360, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 27652784128, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 27652788224, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.40.attn_output.weight", "offset": 27658555392, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.40.attn_q.bias", "offset": 27704692736, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 27704725504, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.40.attn_v.bias", "offset": 27750862848, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 27750866944, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_k.bias", "offset": 27756634112, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 27756638208, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_output.weight", "offset": 27762405376, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.41.attn_q.bias", "offset": 27808542720, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 27808575488, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.41.attn_v.bias", "offset": 27854712832, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 27854716928, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 27860484096, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 27860516864, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.41.ffn_gate.weight", "offset": 28042182656, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.41.ffn_up.weight", "offset": 28208709632, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.41.ffn_norm.weight", "offset": 28375236608, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 28375269376, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 28375302144, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.42.ffn_gate.weight", "offset": 28632662016, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.42.ffn_up.weight", "offset": 28799188992, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.42.ffn_norm.weight", "offset": 28965715968, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 28965748736, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 28965752832, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.42.attn_output.weight", "offset": 28971520000, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.42.attn_q.bias", "offset": 29017657344, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 29017690112, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.42.attn_v.bias", "offset": 29063827456, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 29063831552, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.43.ffn_gate.weight", "offset": 29070712832, "shape": [ 8192, 29568 ], "size": 6881280, "type": 13 }, { "name": "blk.43.attn_k.bias", "offset": 29237239808, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 29237243904, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.attn_output.weight", "offset": 29243011072, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.43.attn_q.bias", "offset": 29289148416, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 29289181184, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.43.attn_v.bias", "offset": 29335318528, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 29335322624, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 29341089792, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 29341122560, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.43.ffn_up.weight", "offset": 29522788352, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.43.ffn_norm.weight", "offset": 29689315328, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 29689348096, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 29689380864, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.44.ffn_gate.weight", "offset": 29871046656, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.44.ffn_up.weight", "offset": 30037573632, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.44.ffn_norm.weight", "offset": 30204100608, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 30204133376, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 30204137472, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.44.attn_output.weight", "offset": 30209904640, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.44.attn_q.bias", "offset": 30256041984, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 30256074752, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.44.attn_v.bias", "offset": 30302212096, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 30302216192, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.ffn_gate.weight", "offset": 30307983360, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.45.ffn_up.weight", "offset": 30474510336, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.45.attn_k.bias", "offset": 30641037312, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 30641041408, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.attn_output.weight", "offset": 30646808576, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.45.attn_q.bias", "offset": 30692945920, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 30692978688, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.45.attn_v.bias", "offset": 30739116032, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 30739120128, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.45.attn_norm.weight", "offset": 30746001408, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 30746034176, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.45.ffn_norm.weight", "offset": 31003394048, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 31003426816, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 31003459584, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.46.ffn_gate.weight", "offset": 31185125376, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.46.ffn_up.weight", "offset": 31351652352, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.46.ffn_norm.weight", "offset": 31518179328, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 31518212096, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 31518216192, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.46.attn_output.weight", "offset": 31523983360, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "lk.58.attn_output.", "offset": 31570120704, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 31570153472, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.46.attn_v.bias", "offset": 31616290816, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 31616294912, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 31622062080, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 31622094848, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.47.ffn_gate.weight", "offset": 31803760640, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.47.ffn_up.weight", "offset": 31970287616, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.47.ffn_norm.weight", "offset": 32136814592, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 32136847360, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 32136851456, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.47.attn_output.weight", "offset": 32142618624, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.47.attn_q.bias", "offset": 32188755968, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 32188788736, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.47.attn_v.bias", "offset": 32234926080, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 32234930176, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.48.attn_norm.weight", "offset": 32240697344, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 32240730112, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.48.ffn_gate.weight", "offset": 32498089984, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.48.ffn_up.weight", "offset": 32664616960, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.48.ffn_norm.weight", "offset": 32831143936, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 32831176704, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 32831180800, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.48.attn_output.weight", "offset": 32836947968, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.48.attn_q.bias", "offset": 32883085312, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 32883118080, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.48.attn_v.bias", "offset": 32929255424, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 32929259520, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.49.attn_norm.weight", "offset": 32936140800, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 32936173568, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.49.ffn_gate.weight", "offset": 33117839360, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.49.ffn_up.weight", "offset": 33284366336, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.49.ffn_norm.weight", "offset": 33450893312, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 33450926080, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 33450930176, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.49.attn_output.weight", "offset": 33456697344, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.49.attn_q.bias", "offset": 33502834688, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 33502867456, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.49.attn_v.bias", "offset": 33549004800, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 33549008896, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_k.bias", "offset": 33554776064, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 33554780160, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_output.weight", "offset": 33560547328, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.50.attn_q.bias", "offset": 33606684672, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 33606717440, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.50.attn_v.bias", "offset": 33652854784, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 33652858880, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_norm.weight", "offset": 33658626048, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 33658658816, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.50.ffn_gate.weight", "offset": 33840324608, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.50.ffn_up.weight", "offset": 34006851584, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.50.ffn_norm.weight", "offset": 34173378560, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 34173411328, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 34173444096, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.51.ffn_gate.weight", "offset": 34430803968, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.51.ffn_up.weight", "offset": 34597330944, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.51.ffn_norm.weight", "offset": 34763857920, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 34763890688, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 34763894784, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.51.attn_output.weight", "offset": 34769661952, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.51.attn_q.bias", "offset": 34815799296, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 34815832064, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.51.attn_v.bias", "offset": 34861969408, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 34861973504, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.52.ffn_gate.weight", "offset": 34868854784, "shape": [ 8192, 29568 ], "size": 6881280, "type": 13 }, { "name": "blk.52.attn_k.bias", "offset": 35035381760, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 35035385856, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.attn_output.weight", "offset": 35041153024, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.52.attn_q.bias", "offset": 35087290368, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 35087323136, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.52.attn_v.bias", "offset": 35133460480, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 35133464576, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.attn_norm.weight", "offset": 35139231744, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 35139264512, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.52.ffn_up.weight", "offset": 35320930304, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.52.ffn_norm.weight", "offset": 35487457280, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 35487490048, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 35487522816, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.53.ffn_gate.weight", "offset": 35669188608, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.53.ffn_up.weight", "offset": 35835715584, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.53.ffn_norm.weight", "offset": 36002242560, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 36002275328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 36002279424, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.53.attn_output.weight", "offset": 36008046592, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.53.attn_q.bias", "offset": 36054183936, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 36054216704, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.53.attn_v.bias", "offset": 36100354048, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 36100358144, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.ffn_gate.weight", "offset": 36106125312, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.54.ffn_up.weight", "offset": 36272652288, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.54.attn_k.bias", "offset": 36439179264, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 36439183360, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.attn_output.weight", "offset": 36444950528, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.54.attn_q.bias", "offset": 36491087872, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 36491120640, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.54.attn_v.bias", "offset": 36537257984, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 36537262080, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.54.attn_norm.weight", "offset": 36544143360, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 36544176128, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.54.ffn_norm.weight", "offset": 36801536000, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 36801568768, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 36801601536, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.55.ffn_gate.weight", "offset": 36983267328, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.55.ffn_up.weight", "offset": 37149794304, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.55.ffn_norm.weight", "offset": 37316321280, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 37316354048, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 37316358144, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.55.attn_output.weight", "offset": 37322125312, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.55.attn_q.bias", "offset": 37368262656, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 37368295424, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.55.attn_v.bias", "offset": 37414432768, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 37414436864, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.56.attn_norm.weight", "offset": 37420204032, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 37420236800, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.56.ffn_gate.weight", "offset": 37601902592, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.56.ffn_up.weight", "offset": 37768429568, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.56.ffn_norm.weight", "offset": 37934956544, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 37934989312, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 37934993408, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.56.attn_output.weight", "offset": 37940760576, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.56.attn_q.bias", "offset": 37986897920, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 37986930688, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.56.attn_v.bias", "offset": 38033068032, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 38033072128, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.57.attn_norm.weight", "offset": 38038839296, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 38038872064, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.57.ffn_gate.weight", "offset": 38296231936, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.57.ffn_up.weight", "offset": 38462758912, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.57.ffn_norm.weight", "offset": 38629285888, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 38629318656, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 38629322752, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.57.attn_output.weight", "offset": 38635089920, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.57.attn_q.bias", "offset": 38681227264, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 38681260032, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.57.attn_v.bias", "offset": 38727397376, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 38727401472, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.58.attn_norm.weight", "offset": 38734282752, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 38734315520, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.58.ffn_gate.weight", "offset": 38915981312, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.58.ffn_up.weight", "offset": 39082508288, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.58.ffn_norm.weight", "offset": 39249035264, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 39249068032, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 39249072128, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.58.attn_output.weight", "offset": 39254839296, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.58.attn_q.bias", "offset": 39300976640, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 39301009408, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.58.attn_v.bias", "offset": 39347146752, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 39347150848, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_k.bias", "offset": 39352918016, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 39352922112, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_output.weight", "offset": 39358689280, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.59.attn_q.bias", "offset": 39404826624, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 39404859392, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.59.attn_v.bias", "offset": 39450996736, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 39451000832, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_norm.weight", "offset": 39456768000, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 39456800768, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.59.ffn_gate.weight", "offset": 39638466560, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.59.ffn_up.weight", "offset": 39804993536, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.59.ffn_norm.weight", "offset": 39971520512, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 39971553280, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 39971586048, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.60.ffn_gate.weight", "offset": 40228945920, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.60.ffn_up.weight", "offset": 40395472896, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.60.ffn_norm.weight", "offset": 40561999872, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 40562032640, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 40562036736, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.60.attn_output.weight", "offset": 40567803904, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.60.attn_q.bias", "offset": 40613941248, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 40613974016, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.60.attn_v.bias", "offset": 40660111360, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 40660115456, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.61.ffn_gate.weight", "offset": 40666996736, "shape": [ 8192, 29568 ], "size": 6881280, "type": 13 }, { "name": "blk.61.attn_k.bias", "offset": 40833523712, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 40833527808, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.attn_output.weight", "offset": 40839294976, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.61.attn_q.bias", "offset": 40885432320, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 40885465088, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.61.attn_v.bias", "offset": 40931602432, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 40931606528, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.attn_norm.weight", "offset": 40937373696, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 40937406464, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.61.ffn_up.weight", "offset": 41119072256, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.61.ffn_norm.weight", "offset": 41285599232, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 41285632000, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 41285664768, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.62.ffn_gate.weight", "offset": 41467330560, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.62.ffn_up.weight", "offset": 41633857536, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.62.ffn_norm.weight", "offset": 41800384512, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 41800417280, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 41800421376, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.62.attn_output.weight", "offset": 41806188544, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.62.attn_q.bias", "offset": 41852325888, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 41852358656, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.62.attn_v.bias", "offset": 41898496000, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 41898500096, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.ffn_gate.weight", "offset": 41904267264, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.63.ffn_up.weight", "offset": 42070794240, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.63.attn_k.bias", "offset": 42237321216, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 42237325312, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.attn_output.weight", "offset": 42243092480, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.63.attn_q.bias", "offset": 42289229824, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 42289262592, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.63.attn_v.bias", "offset": 42335399936, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 42335404032, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.63.attn_norm.weight", "offset": 42342285312, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 42342318080, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.63.ffn_norm.weight", "offset": 42599677952, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 42599710720, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 42599743488, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.64.ffn_gate.weight", "offset": 42781409280, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.64.ffn_up.weight", "offset": 42947936256, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.64.ffn_norm.weight", "offset": 43114463232, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 43114496000, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 43114500096, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.64.attn_output.weight", "offset": 43120267264, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.64.attn_q.bias", "offset": 43166404608, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 43166437376, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.64.attn_v.bias", "offset": 43212574720, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 43212578816, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.65.attn_norm.weight", "offset": 43218345984, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 43218378752, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.65.ffn_gate.weight", "offset": 43400044544, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.65.ffn_up.weight", "offset": 43566571520, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.65.ffn_norm.weight", "offset": 43733098496, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 43733131264, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 43733135360, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.65.attn_output.weight", "offset": 43738902528, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.65.attn_q.bias", "offset": 43785039872, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 43785072640, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.65.attn_v.bias", "offset": 43831209984, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 43831214080, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.66.attn_norm.weight", "offset": 43836981248, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 43837014016, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.66.ffn_gate.weight", "offset": 44094373888, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.66.ffn_up.weight", "offset": 44260900864, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.66.ffn_norm.weight", "offset": 44427427840, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 44427460608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 44427464704, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.66.attn_output.weight", "offset": 44433231872, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.66.attn_q.bias", "offset": 44479369216, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 44479401984, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.66.attn_v.bias", "offset": 44525539328, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 44525543424, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.67.attn_norm.weight", "offset": 44532424704, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 44532457472, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.67.ffn_gate.weight", "offset": 44714123264, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.67.ffn_up.weight", "offset": 44880650240, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.67.ffn_norm.weight", "offset": 45047177216, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 45047209984, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 45047214080, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.67.attn_output.weight", "offset": 45052981248, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.67.attn_q.bias", "offset": 45099118592, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 45099151360, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.67.attn_v.bias", "offset": 45145288704, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 45145292800, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_k.bias", "offset": 45151059968, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 45151064064, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_output.weight", "offset": 45156831232, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.68.attn_q.bias", "offset": 45202968576, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 45203001344, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.68.attn_v.bias", "offset": 45249138688, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 45249142784, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_norm.weight", "offset": 45254909952, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 45254942720, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.68.ffn_gate.weight", "offset": 45436608512, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.68.ffn_up.weight", "offset": 45603135488, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.68.ffn_norm.weight", "offset": 45769662464, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 45769695232, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 45769728000, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.69.ffn_gate.weight", "offset": 46027087872, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.69.ffn_up.weight", "offset": 46193614848, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.69.ffn_norm.weight", "offset": 46360141824, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 46360174592, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 46360178688, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.69.attn_output.weight", "offset": 46365945856, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.69.attn_q.bias", "offset": 46412083200, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 46412115968, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.69.attn_v.bias", "offset": 46458253312, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 46458257408, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.70.ffn_gate.weight", "offset": 46465138688, "shape": [ 8192, 29568 ], "size": 6881280, "type": 13 }, { "name": "blk.70.attn_k.bias", "offset": 46631665664, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 46631669760, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.attn_output.weight", "offset": 46637436928, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.70.attn_q.bias", "offset": 46683574272, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 46683607040, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "\u0014\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.70.att", "offset": 46729744384, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 46729748480, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.70.attn_norm.weight", "offset": 46736629760, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 46736662528, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.70.ffn_up.weight", "offset": 46994022400, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.70.ffn_norm.weight", "offset": 47160549376, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 47160582144, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 47160614912, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.71.ffn_gate.weight", "offset": 47417974784, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.71.ffn_up.weight", "offset": 47584501760, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.71.ffn_norm.weight", "offset": 47751028736, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 47751061504, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 47751065600, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.71.attn_output.weight", "offset": 47756832768, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.71.attn_q.bias", "offset": 47802970112, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 47803002880, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.71.attn_v.bias", "offset": 47849140224, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 47849144320, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.72.ffn_gate.weight", "offset": 47856025600, "shape": [ 8192, 29568 ], "size": 6881280, "type": 13 }, { "name": "blk.72.ffn_up.weight", "offset": 48022552576, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.72.attn_k.bias", "offset": 48189079552, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 48189083648, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.attn_output.weight", "offset": 48194850816, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.72.attn_q.bias", "offset": 48240988160, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 48241020928, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.72.attn_v.bias", "offset": 48287158272, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 48287162368, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.72.attn_norm.weight", "offset": 48294043648, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 48294076416, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.72.ffn_norm.weight", "offset": 48551436288, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 48551469056, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 48551501824, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.73.ffn_gate.weight", "offset": 48808861696, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.73.ffn_up.weight", "offset": 48975388672, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.73.ffn_norm.weight", "offset": 49141915648, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 49141948416, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 49141952512, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.73.attn_output.weight", "offset": 49147719680, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.73.attn_q.bias", "offset": 49193857024, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 49193889792, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.73.attn_v.bias", "offset": 49240027136, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 49240031232, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.74.attn_norm.weight", "offset": 49246912512, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 49246945280, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.74.ffn_gate.weight", "offset": 49504305152, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.74.ffn_up.weight", "offset": 49670832128, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.74.ffn_norm.weight", "offset": 49837359104, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 49837391872, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 49837395968, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.74.attn_output.weight", "offset": 49843163136, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.74.attn_q.bias", "offset": 49889300480, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 49889333248, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.74.attn_v.bias", "offset": 49935470592, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 49935474688, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.75.attn_norm.weight", "offset": 49942355968, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 49942388736, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.75.ffn_gate.weight", "offset": 50199748608, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.75.ffn_up.weight", "offset": 50366275584, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.75.ffn_norm.weight", "offset": 50532802560, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 50532835328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 50532839424, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.75.attn_output.weight", "offset": 50538606592, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.75.attn_q.bias", "offset": 50584743936, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 50584776704, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.75.attn_v.bias", "offset": 50630914048, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 50630918144, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.76.attn_norm.weight", "offset": 50637799424, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 50637832192, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.76.ffn_gate.weight", "offset": 50895192064, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.76.ffn_up.weight", "offset": 51061719040, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.76.ffn_norm.weight", "offset": 51228246016, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 51228278784, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 51228282880, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.76.attn_output.weight", "offset": 51234050048, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.76.attn_q.bias", "offset": 51280187392, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 51280220160, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.76.attn_v.bias", "offset": 51326357504, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 51326361600, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.77.attn_k.bias", "offset": 51333242880, "shape": [ 1024 ], "size": 6881280, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 51333246976, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_output.weight", "offset": 51339014144, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.77.attn_q.bias", "offset": 51385151488, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 51385184256, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.77.attn_v.bias", "offset": 51431321600, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 51431325696, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.77.attn_norm.weight", "offset": 51438206976, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 51438239744, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.77.ffn_gate.weight", "offset": 51695599616, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.77.ffn_up.weight", "offset": 51862126592, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.77.ffn_norm.weight", "offset": 52028653568, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 52028686336, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 52028719104, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.78.ffn_gate.weight", "offset": 52286078976, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.78.ffn_up.weight", "offset": 52452605952, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.78.ffn_norm.weight", "offset": 52619132928, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 52619165696, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 52619169792, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.78.attn_output.weight", "offset": 52624936960, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.78.attn_q.bias", "offset": 52671074304, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 52671107072, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.78.attn_v.bias", "offset": 52717244416, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 52717248512, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.79.ffn_gate.weight", "offset": 52724129792, "shape": [ 8192, 29568 ], "size": 6881280, "type": 13 }, { "name": "blk.79.attn_k.bias", "offset": 52890656768, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 52890660864, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.79.attn_output.weight", "offset": 52896428032, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.79.attn_q.bias", "offset": 52942565376, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 52942598144, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.79.attn_v.bias", "offset": 52988735488, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 52988739584, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "output.weight", "offset": 52995620864, "shape": [ 8192, 152064 ], "size": 6881280, "type": 14 }, { "name": "blk.79.attn_norm.weight", "offset": 54017490944, "shape": [ 8192 ], "size": 1021870080, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 54017523712, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.79.ffn_up.weight", "offset": 54274883584, "shape": [ 8192, 29568 ], "size": 257359872, "type": 13 }, { "name": "blk.79.ffn_norm.weight", "offset": 54441410560, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "output_norm.weight", "offset": 54441443328, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }