qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 9, "general.name": "Qwen2-72B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 80, "qwen2.context_length": 32768, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 934281216, "shape": [ 8192, 29568 ], "size": 934281216, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 1115947008, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.0.attn_k.bias", "offset": 1297612800, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 1297616896, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.0.attn_output.weight", "offset": 1303908352, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.0.attn_q.bias", "offset": 1354240000, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 1354272768, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.0.attn_v.bias", "offset": 1404604416, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 1404608512, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.0.attn_norm.weight", "offset": 1410899968, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 1410932736, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 1592598528, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1592631296, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1592664064, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.1.ffn_gate.weight", "offset": 1774329856, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.1.ffn_up.weight", "offset": 1955995648, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.1.ffn_norm.weight", "offset": 2137661440, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 2137694208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 2137698304, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.1.attn_output.weight", "offset": 2143989760, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.1.attn_q.bias", "offset": 2194321408, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 2194354176, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.1.attn_v.bias", "offset": 2244685824, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 2244689920, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 2250981376, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 2251014144, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.2.ffn_gate.weight", "offset": 2432679936, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.2.ffn_up.weight", "offset": 2614345728, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.2.ffn_norm.weight", "offset": 2796011520, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 2796044288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 2796048384, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.2.attn_output.weight", "offset": 2802339840, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.2.attn_q.bias", "offset": 2852671488, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 2852704256, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.2.attn_v.bias", "offset": 2903035904, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 2903040000, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 2909331456, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 2909364224, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.3.ffn_gate.weight", "offset": 3091030016, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.3.ffn_up.weight", "offset": 3272695808, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.3.ffn_norm.weight", "offset": 3454361600, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 3454394368, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 3454398464, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.3.attn_output.weight", "offset": 3460689920, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.3.attn_q.bias", "offset": 3511021568, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 3511054336, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.3.attn_v.bias", "offset": 3561385984, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 3561390080, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 3567681536, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 3567714304, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.4.ffn_gate.weight", "offset": 3749380096, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.4.ffn_up.weight", "offset": 3931045888, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.4.ffn_norm.weight", "offset": 4112711680, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 4112744448, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 4112748544, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.4.attn_output.weight", "offset": 4119040000, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.4.attn_q.bias", "offset": 4169371648, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 4169404416, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.4.attn_v.bias", "offset": 4219736064, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 4219740160, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.5.attn_k.bias", "offset": 4226031616, "shape": [ 1024 ], "size": 6291456, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 4226035712, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.5.attn_output.weight", "offset": 4232327168, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.5.attn_q.bias", "offset": 4282658816, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 4282691584, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.5.attn_v.bias", "offset": 4333023232, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 4333027328, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 4339318784, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 4339351552, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.5.ffn_gate.weight", "offset": 4521017344, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.5.ffn_up.weight", "offset": 4702683136, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.5.ffn_norm.weight", "offset": 4884348928, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 4884381696, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 4884414464, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.6.ffn_gate.weight", "offset": 5066080256, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.6.ffn_up.weight", "offset": 5247746048, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.6.ffn_norm.weight", "offset": 5429411840, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 5429444608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 5429448704, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 5435740160, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.6.attn_q.bias", "offset": 5486071808, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 5486104576, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.6.attn_v.bias", "offset": 5536436224, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 5536440320, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.7.ffn_gate.weight", "offset": 5542731776, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.7.attn_k.bias", "offset": 5724397568, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 5724401664, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.7.attn_output.weight", "offset": 5730693120, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.7.attn_q.bias", "offset": 5781024768, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 5781057536, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.7.attn_v.bias", "offset": 5831389184, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 5831393280, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.7.attn_norm.weight", "offset": 5837684736, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 5837717504, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 6019383296, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.7.ffn_norm.weight", "offset": 6201049088, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 6201081856, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 6201114624, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.8.ffn_gate.weight", "offset": 6382780416, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.8.ffn_up.weight", "offset": 6564446208, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.8.ffn_norm.weight", "offset": 6746112000, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 6746144768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 6746148864, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.8.attn_output.weight", "offset": 6752440320, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.8.attn_q.bias", "offset": 6802771968, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 6802804736, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.8.attn_v.bias", "offset": 6853136384, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 6853140480, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.9.ffn_gate.weight", "offset": 6859431936, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.9.ffn_up.weight", "offset": 7041097728, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.9.attn_k.bias", "offset": 7222763520, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 7222767616, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.9.attn_output.weight", "offset": 7229059072, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.9.attn_q.bias", "offset": 7279390720, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 7279423488, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.9.attn_v.bias", "offset": 7329755136, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 7329759232, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 7336050688, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 7336083456, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.10.ffn_gate.weight", "offset": 7517749248, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.10.ffn_up.weight", "offset": 7699415040, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 7881080832, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 7881113600, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 7881117696, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.10.attn_output.weight", "offset": 7887409152, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.10.attn_q.bias", "offset": 7937740800, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 7937773568, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.10.attn_v.bias", "offset": 7988105216, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 7988109312, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 7994400768, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 7994433536, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.11.ffn_gate.weight", "offset": 8176099328, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.11.ffn_up.weight", "offset": 8357765120, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.11.ffn_norm.weight", "offset": 8539430912, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 8539463680, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 8539467776, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.11.attn_output.weight", "offset": 8545759232, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.11.attn_q.bias", "offset": 8596090880, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 8596123648, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.11.attn_v.bias", "offset": 8646455296, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 8646459392, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 8652750848, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 8652783616, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 8834449408, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 8834482176, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 8834514944, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.12.ffn_gate.weight", "offset": 9016180736, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.12.ffn_up.weight", "offset": 9197846528, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 9379512320, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 9379545088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 9379549184, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.12.attn_output.weight", "offset": 9385840640, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.12.attn_q.bias", "offset": 9436172288, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 9436205056, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.12.attn_v.bias", "offset": 9486536704, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 9486540800, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 9492832256, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 9492865024, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.13.ffn_gate.weight", "offset": 9674530816, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.13.ffn_up.weight", "offset": 9856196608, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 10037862400, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 10037895168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 10037899264, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.13.attn_output.weight", "offset": 10044190720, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.13.attn_q.bias", "offset": 10094522368, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 10094555136, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.13.attn_v.bias", "offset": 10144886784, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 10144890880, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.14.attn_k.bias", "offset": 10151182336, "shape": [ 1024 ], "size": 6291456, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 10151186432, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.14.attn_output.weight", "offset": 10157477888, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.14.attn_q.bias", "offset": 10207809536, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 10207842304, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.14.attn_v.bias", "offset": 10258173952, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 10258178048, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 10264469504, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 10264502272, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.14.ffn_gate.weight", "offset": 10446168064, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.14.ffn_up.weight", "offset": 10627833856, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 10809499648, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 10809532416, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 10809565184, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.15.ffn_gate.weight", "offset": 10991230976, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.15.ffn_up.weight", "offset": 11172896768, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.15.ffn_norm.weight", "offset": 11354562560, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 11354595328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 11354599424, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.15.attn_output.weight", "offset": 11360890880, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.15.attn_q.bias", "offset": 11411222528, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 11411255296, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.15.attn_v.bias", "offset": 11461586944, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 11461591040, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.16.ffn_gate.weight", "offset": 11467882496, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.16.attn_k.bias", "offset": 11649548288, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 11649552384, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.16.attn_output.weight", "offset": 11655843840, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.16.attn_q.bias", "offset": 11706175488, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 11706208256, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.16.attn_v.bias", "offset": 11756539904, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 11756544000, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 11762835456, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 11762868224, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 11944534016, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 12126199808, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 12126232576, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 12126265344, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.17.ffn_gate.weight", "offset": 12307931136, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.17.ffn_up.weight", "offset": 12489596928, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.17.ffn_norm.weight", "offset": 12671262720, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 12671295488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 12671299584, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.17.attn_output.weight", "offset": 12677591040, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.17.attn_q.bias", "offset": 12727922688, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 12727955456, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.17.attn_v.bias", "offset": 12778287104, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 12778291200, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.18.ffn_gate.weight", "offset": 12784582656, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.18.ffn_up.weight", "offset": 12966248448, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.18.attn_k.bias", "offset": 13147914240, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 13147918336, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.18.attn_output.weight", "offset": 13154209792, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.18.attn_q.bias", "offset": 13204541440, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 13204574208, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.18.attn_v.bias", "offset": 13254905856, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 13254909952, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 13261201408, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 13261234176, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 13442899968, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 13442932736, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 13442965504, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.19.ffn_gate.weight", "offset": 13624631296, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.19.ffn_up.weight", "offset": 13806297088, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.19.ffn_norm.weight", "offset": 13987962880, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 13987995648, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 13987999744, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.19.attn_output.weight", "offset": 13994291200, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.19.attn_q.bias", "offset": 14044622848, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 14044655616, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.19.attn_v.bias", "offset": 14094987264, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 14094991360, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 14101282816, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 14101315584, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.20.ffn_gate.weight", "offset": 14282981376, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.20.ffn_up.weight", "offset": 14464647168, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.20.ffn_norm.weight", "offset": 14646312960, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 14646345728, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 14646349824, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.20.attn_output.weight", "offset": 14652641280, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.20.attn_q.bias", "offset": 14702972928, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 14703005696, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.20.attn_v.bias", "offset": 14753337344, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 14753341440, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 14759632896, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 14759665664, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.21.ffn_gate.weight", "offset": 14941331456, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.21.ffn_up.weight", "offset": 15122997248, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.21.ffn_norm.weight", "offset": 15304663040, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 15304695808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 15304699904, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 15310991360, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.21.attn_q.bias", "offset": 15361323008, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 15361355776, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.21.attn_v.bias", "offset": 15411687424, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 15411691520, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.22.attn_norm.weight", "offset": 15417982976, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 15418015744, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.22.ffn_gate.weight", "offset": 15599681536, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.22.ffn_up.weight", "offset": 15781347328, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.22.ffn_norm.weight", "offset": 15963013120, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 15963045888, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 15963049984, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.22.attn_output.weight", "offset": 15969341440, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.22.attn_q.bias", "offset": 16019673088, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 16019705856, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.22.attn_v.bias", "offset": 16070037504, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 16070041600, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.23.attn_k.bias", "offset": 16076333056, "shape": [ 1024 ], "size": 6291456, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 16076337152, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.23.attn_output.weight", "offset": 16082628608, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.23.attn_q.bias", "offset": 16132960256, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 16132993024, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.23.attn_v.bias", "offset": 16183324672, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 16183328768, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 16189620224, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 16189652992, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.23.ffn_gate.weight", "offset": 16371318784, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.23.ffn_up.weight", "offset": 16552984576, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.23.ffn_norm.weight", "offset": 16734650368, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 16734683136, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 16734715904, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.24.ffn_gate.weight", "offset": 16916381696, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.24.ffn_up.weight", "offset": 17098047488, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.24.ffn_norm.weight", "offset": 17279713280, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 17279746048, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 17279750144, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.24.attn_output.weight", "offset": 17286041600, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.24.attn_q.bias", "offset": 17336373248, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 17336406016, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.24.attn_v.bias", "offset": 17386737664, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 17386741760, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.25.ffn_gate.weight", "offset": 17393033216, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.25.attn_k.bias", "offset": 17574699008, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 17574703104, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.25.attn_output.weight", "offset": 17580994560, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.25.attn_q.bias", "offset": 17631326208, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 17631358976, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.25.attn_v.bias", "offset": 17681690624, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 17681694720, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.25.attn_norm.weight", "offset": 17687986176, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 17688018944, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.25.ffn_up.weight", "offset": 17869684736, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.25.ffn_norm.weight", "offset": 18051350528, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 18051383296, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 18051416064, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.26.ffn_gate.weight", "offset": 18233081856, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.26.ffn_up.weight", "offset": 18414747648, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.26.ffn_norm.weight", "offset": 18596413440, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 18596446208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 18596450304, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.26.attn_output.weight", "offset": 18602741760, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.26.attn_q.bias", "offset": 18653073408, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 18653106176, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.26.attn_v.bias", "offset": 18703437824, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 18703441920, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.27.ffn_gate.weight", "offset": 18709733376, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.27.ffn_up.weight", "offset": 18891399168, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.27.attn_k.bias", "offset": 19073064960, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 19073069056, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.27.attn_output.weight", "offset": 19079360512, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.27.attn_q.bias", "offset": 19129692160, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 19129724928, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.27.attn_v.bias", "offset": 19180056576, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 19180060672, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.27.attn_norm.weight", "offset": 19186352128, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 19186384896, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.27.ffn_norm.weight", "offset": 19368050688, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 19368083456, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 19368116224, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.28.ffn_gate.weight", "offset": 19549782016, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.28.ffn_up.weight", "offset": 19731447808, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.28.ffn_norm.weight", "offset": 19913113600, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 19913146368, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 19913150464, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.28.attn_output.weight", "offset": 19919441920, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.28.attn_q.bias", "offset": 19969773568, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 19969806336, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.28.attn_v.bias", "offset": 20020137984, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 20020142080, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.29.attn_norm.weight", "offset": 20026433536, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 20026466304, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.29.ffn_gate.weight", "offset": 20208132096, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.29.ffn_up.weight", "offset": 20389797888, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.29.ffn_norm.weight", "offset": 20571463680, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 20571496448, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 20571500544, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.29.attn_output.weight", "offset": 20577792000, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.29.attn_q.bias", "offset": 20628123648, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 20628156416, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.29.attn_v.bias", "offset": 20678488064, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 20678492160, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.30.attn_norm.weight", "offset": 20684783616, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 20684816384, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.30.ffn_gate.weight", "offset": 20866482176, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.30.ffn_up.weight", "offset": 21048147968, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.30.ffn_norm.weight", "offset": 21229813760, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 21229846528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 21229850624, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.30.attn_output.weight", "offset": 21236142080, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.30.attn_q.bias", "offset": 21286473728, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 21286506496, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.30.attn_v.bias", "offset": 21336838144, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 21336842240, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.31.attn_norm.weight", "offset": 21343133696, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 21343166464, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.31.ffn_gate.weight", "offset": 21524832256, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.31.ffn_up.weight", "offset": 21706498048, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.31.ffn_norm.weight", "offset": 21888163840, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 21888196608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 21888200704, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.31.attn_output.weight", "offset": 21894492160, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.31.attn_q.bias", "offset": 21944823808, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 21944856576, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.31.attn_v.bias", "offset": 21995188224, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 21995192320, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.32.attn_k.bias", "offset": 22001483776, "shape": [ 1024 ], "size": 6291456, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 22001487872, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.32.attn_output.weight", "offset": 22007779328, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.32.attn_q.bias", "offset": 22058110976, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 22058143744, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.32.attn_v.bias", "offset": 22108475392, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 22108479488, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.32.attn_norm.weight", "offset": 22114770944, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 22114803712, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.32.ffn_gate.weight", "offset": 22296469504, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.32.ffn_up.weight", "offset": 22478135296, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.32.ffn_norm.weight", "offset": 22659801088, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 22659833856, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 22659866624, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.33.ffn_gate.weight", "offset": 22841532416, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.33.ffn_up.weight", "offset": 23023198208, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.33.ffn_norm.weight", "offset": 23204864000, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 23204896768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 23204900864, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.33.attn_output.weight", "offset": 23211192320, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.33.attn_q.bias", "offset": 23261523968, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 23261556736, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.33.attn_v.bias", "offset": 23311888384, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 23311892480, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.34.ffn_gate.weight", "offset": 23318183936, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.34.attn_k.bias", "offset": 23499849728, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 23499853824, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.34.attn_output.weight", "offset": 23506145280, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.34.attn_q.bias", "offset": 23556476928, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 23556509696, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.34.attn_v.bias", "offset": 23606841344, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 23606845440, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.34.attn_norm.weight", "offset": 23613136896, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 23613169664, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.34.ffn_up.weight", "offset": 23794835456, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.34.ffn_norm.weight", "offset": 23976501248, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 23976534016, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 23976566784, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.35.ffn_gate.weight", "offset": 24158232576, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.35.ffn_up.weight", "offset": 24339898368, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.35.ffn_norm.weight", "offset": 24521564160, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 24521596928, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 24521601024, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.35.attn_output.weight", "offset": 24527892480, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.35.attn_q.bias", "offset": 24578224128, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 24578256896, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.35.attn_v.bias", "offset": 24628588544, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 24628592640, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.36.ffn_gate.weight", "offset": 24634884096, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.36.ffn_up.weight", "offset": 24816549888, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.36.attn_k.bias", "offset": 24998215680, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 24998219776, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.36.attn_output.weight", "offset": 25004511232, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.36.attn_q.bias", "offset": 25054842880, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 25054875648, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.36.attn_v.bias", "offset": 25105207296, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 25105211392, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.36.attn_norm.weight", "offset": 25111502848, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 25111535616, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.36.ffn_norm.weight", "offset": 25293201408, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 25293234176, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 25293266944, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.37.ffn_gate.weight", "offset": 25474932736, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.37.ffn_up.weight", "offset": 25656598528, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.37.ffn_norm.weight", "offset": 25838264320, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 25838297088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 25838301184, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.37.attn_output.weight", "offset": 25844592640, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.37.attn_q.bias", "offset": 25894924288, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 25894957056, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.37.attn_v.bias", "offset": 25945288704, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 25945292800, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.38.attn_norm.weight", "offset": 25951584256, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 25951617024, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.38.ffn_gate.weight", "offset": 26133282816, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.38.ffn_up.weight", "offset": 26314948608, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.38.ffn_norm.weight", "offset": 26496614400, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 26496647168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 26496651264, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.38.attn_output.weight", "offset": 26502942720, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.38.attn_q.bias", "offset": 26553274368, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 26553307136, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.38.attn_v.bias", "offset": 26603638784, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 26603642880, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.39.attn_norm.weight", "offset": 26609934336, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 26609967104, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.39.ffn_gate.weight", "offset": 26791632896, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.39.ffn_up.weight", "offset": 26973298688, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.39.ffn_norm.weight", "offset": 27154964480, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 27154997248, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 27155001344, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.39.attn_output.weight", "offset": 27161292800, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.39.attn_q.bias", "offset": 27211624448, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 27211657216, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.39.attn_v.bias", "offset": 27261988864, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 27261992960, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.40.attn_norm.weight", "offset": 27268284416, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 27268317184, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.40.ffn_gate.weight", "offset": 27449982976, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.40.ffn_up.weight", "offset": 27631648768, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.40.ffn_norm.weight", "offset": 27813314560, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 27813347328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 27813351424, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.40.attn_output.weight", "offset": 27819642880, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.40.attn_q.bias", "offset": 27869974528, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 27870007296, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.40.attn_v.bias", "offset": 27920338944, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 27920343040, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.41.attn_k.bias", "offset": 27926634496, "shape": [ 1024 ], "size": 6291456, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 27926638592, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.41.attn_output.weight", "offset": 27932930048, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.41.attn_q.bias", "offset": 27983261696, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 27983294464, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.41.attn_v.bias", "offset": 28033626112, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 28033630208, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.41.attn_norm.weight", "offset": 28039921664, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 28039954432, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.41.ffn_gate.weight", "offset": 28221620224, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.41.ffn_up.weight", "offset": 28403286016, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.41.ffn_norm.weight", "offset": 28584951808, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 28584984576, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 28585017344, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.42.ffn_gate.weight", "offset": 28766683136, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.42.ffn_up.weight", "offset": 28948348928, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.42.ffn_norm.weight", "offset": 29130014720, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 29130047488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 29130051584, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.42.attn_output.weight", "offset": 29136343040, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.42.attn_q.bias", "offset": 29186674688, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 29186707456, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.42.attn_v.bias", "offset": 29237039104, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 29237043200, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.43.ffn_gate.weight", "offset": 29243334656, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.43.attn_k.bias", "offset": 29425000448, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 29425004544, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.43.attn_output.weight", "offset": 29431296000, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.43.attn_q.bias", "offset": 29481627648, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 29481660416, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.43.attn_v.bias", "offset": 29531992064, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 29531996160, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.43.attn_norm.weight", "offset": 29538287616, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 29538320384, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.43.ffn_up.weight", "offset": 29719986176, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.43.ffn_norm.weight", "offset": 29901651968, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 29901684736, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 29901717504, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.44.ffn_gate.weight", "offset": 30083383296, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.44.ffn_up.weight", "offset": 30265049088, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.44.ffn_norm.weight", "offset": 30446714880, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 30446747648, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 30446751744, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.44.attn_output.weight", "offset": 30453043200, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.44.attn_q.bias", "offset": 30503374848, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 30503407616, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.44.attn_v.bias", "offset": 30553739264, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 30553743360, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.45.ffn_gate.weight", "offset": 30560034816, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.45.ffn_up.weight", "offset": 30741700608, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.45.attn_k.bias", "offset": 30923366400, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 30923370496, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.45.attn_output.weight", "offset": 30929661952, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.45.attn_q.bias", "offset": 30979993600, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 30980026368, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.45.attn_v.bias", "offset": 31030358016, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 31030362112, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.45.attn_norm.weight", "offset": 31036653568, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 31036686336, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.45.ffn_norm.weight", "offset": 31218352128, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 31218384896, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 31218417664, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.46.ffn_gate.weight", "offset": 31400083456, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.46.ffn_up.weight", "offset": 31581749248, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.46.ffn_norm.weight", "offset": 31763415040, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 31763447808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 31763451904, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.46.attn_output.weight", "offset": 31769743360, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "\u0000\u0000\u0000blk.58.attn_out", "offset": 31820075008, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 31820107776, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.46.attn_v.bias", "offset": 31870439424, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 31870443520, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.47.attn_norm.weight", "offset": 31876734976, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 31876767744, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.47.ffn_gate.weight", "offset": 32058433536, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.47.ffn_up.weight", "offset": 32240099328, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.47.ffn_norm.weight", "offset": 32421765120, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 32421797888, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 32421801984, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.47.attn_output.weight", "offset": 32428093440, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.47.attn_q.bias", "offset": 32478425088, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 32478457856, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.47.attn_v.bias", "offset": 32528789504, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 32528793600, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.48.attn_norm.weight", "offset": 32535085056, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 32535117824, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.48.ffn_gate.weight", "offset": 32716783616, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.48.ffn_up.weight", "offset": 32898449408, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.48.ffn_norm.weight", "offset": 33080115200, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 33080147968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 33080152064, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.48.attn_output.weight", "offset": 33086443520, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.48.attn_q.bias", "offset": 33136775168, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 33136807936, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.48.attn_v.bias", "offset": 33187139584, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 33187143680, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.49.attn_norm.weight", "offset": 33193435136, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 33193467904, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.49.ffn_gate.weight", "offset": 33375133696, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.49.ffn_up.weight", "offset": 33556799488, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.49.ffn_norm.weight", "offset": 33738465280, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 33738498048, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 33738502144, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.49.attn_output.weight", "offset": 33744793600, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.49.attn_q.bias", "offset": 33795125248, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 33795158016, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.49.attn_v.bias", "offset": 33845489664, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 33845493760, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.50.attn_k.bias", "offset": 33851785216, "shape": [ 1024 ], "size": 6291456, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 33851789312, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.50.attn_output.weight", "offset": 33858080768, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.50.attn_q.bias", "offset": 33908412416, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 33908445184, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.50.attn_v.bias", "offset": 33958776832, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 33958780928, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.50.attn_norm.weight", "offset": 33965072384, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 33965105152, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.50.ffn_gate.weight", "offset": 34146770944, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.50.ffn_up.weight", "offset": 34328436736, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.50.ffn_norm.weight", "offset": 34510102528, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 34510135296, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 34510168064, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.51.ffn_gate.weight", "offset": 34691833856, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.51.ffn_up.weight", "offset": 34873499648, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.51.ffn_norm.weight", "offset": 35055165440, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 35055198208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 35055202304, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.51.attn_output.weight", "offset": 35061493760, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.51.attn_q.bias", "offset": 35111825408, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 35111858176, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.51.attn_v.bias", "offset": 35162189824, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 35162193920, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.52.ffn_gate.weight", "offset": 35168485376, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.52.attn_k.bias", "offset": 35350151168, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 35350155264, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.52.attn_output.weight", "offset": 35356446720, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.52.attn_q.bias", "offset": 35406778368, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 35406811136, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.52.attn_v.bias", "offset": 35457142784, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 35457146880, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.52.attn_norm.weight", "offset": 35463438336, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 35463471104, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.52.ffn_up.weight", "offset": 35645136896, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.52.ffn_norm.weight", "offset": 35826802688, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 35826835456, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 35826868224, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.53.ffn_gate.weight", "offset": 36008534016, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.53.ffn_up.weight", "offset": 36190199808, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.53.ffn_norm.weight", "offset": 36371865600, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 36371898368, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 36371902464, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.53.attn_output.weight", "offset": 36378193920, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.53.attn_q.bias", "offset": 36428525568, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 36428558336, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.53.attn_v.bias", "offset": 36478889984, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 36478894080, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.54.ffn_gate.weight", "offset": 36485185536, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.54.ffn_up.weight", "offset": 36666851328, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.54.attn_k.bias", "offset": 36848517120, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 36848521216, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.54.attn_output.weight", "offset": 36854812672, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.54.attn_q.bias", "offset": 36905144320, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 36905177088, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.54.attn_v.bias", "offset": 36955508736, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 36955512832, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.54.attn_norm.weight", "offset": 36961804288, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 36961837056, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.54.ffn_norm.weight", "offset": 37143502848, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 37143535616, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 37143568384, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.55.ffn_gate.weight", "offset": 37325234176, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.55.ffn_up.weight", "offset": 37506899968, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.55.ffn_norm.weight", "offset": 37688565760, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 37688598528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 37688602624, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.55.attn_output.weight", "offset": 37694894080, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.55.attn_q.bias", "offset": 37745225728, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 37745258496, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.55.attn_v.bias", "offset": 37795590144, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 37795594240, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.56.attn_norm.weight", "offset": 37801885696, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 37801918464, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.56.ffn_gate.weight", "offset": 37983584256, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.56.ffn_up.weight", "offset": 38165250048, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.56.ffn_norm.weight", "offset": 38346915840, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 38346948608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 38346952704, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.56.attn_output.weight", "offset": 38353244160, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.56.attn_q.bias", "offset": 38403575808, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 38403608576, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.56.attn_v.bias", "offset": 38453940224, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 38453944320, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.57.attn_norm.weight", "offset": 38460235776, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 38460268544, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.57.ffn_gate.weight", "offset": 38641934336, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.57.ffn_up.weight", "offset": 38823600128, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.57.ffn_norm.weight", "offset": 39005265920, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 39005298688, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 39005302784, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.57.attn_output.weight", "offset": 39011594240, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.57.attn_q.bias", "offset": 39061925888, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 39061958656, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.57.attn_v.bias", "offset": 39112290304, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 39112294400, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.58.attn_norm.weight", "offset": 39118585856, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 39118618624, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.58.ffn_gate.weight", "offset": 39300284416, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.58.ffn_up.weight", "offset": 39481950208, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.58.ffn_norm.weight", "offset": 39663616000, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 39663648768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 39663652864, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.58.attn_output.weight", "offset": 39669944320, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.58.attn_q.bias", "offset": 39720275968, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 39720308736, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.58.attn_v.bias", "offset": 39770640384, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 39770644480, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.59.attn_k.bias", "offset": 39776935936, "shape": [ 1024 ], "size": 6291456, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 39776940032, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.59.attn_output.weight", "offset": 39783231488, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.59.attn_q.bias", "offset": 39833563136, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 39833595904, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.59.attn_v.bias", "offset": 39883927552, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 39883931648, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.59.attn_norm.weight", "offset": 39890223104, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 39890255872, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.59.ffn_gate.weight", "offset": 40071921664, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.59.ffn_up.weight", "offset": 40253587456, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.59.ffn_norm.weight", "offset": 40435253248, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 40435286016, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 40435318784, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.60.ffn_gate.weight", "offset": 40616984576, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.60.ffn_up.weight", "offset": 40798650368, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.60.ffn_norm.weight", "offset": 40980316160, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 40980348928, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 40980353024, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.60.attn_output.weight", "offset": 40986644480, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.60.attn_q.bias", "offset": 41036976128, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 41037008896, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.60.attn_v.bias", "offset": 41087340544, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 41087344640, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.61.ffn_gate.weight", "offset": 41093636096, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.61.attn_k.bias", "offset": 41275301888, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 41275305984, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.61.attn_output.weight", "offset": 41281597440, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.61.attn_q.bias", "offset": 41331929088, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 41331961856, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.61.attn_v.bias", "offset": 41382293504, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 41382297600, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.61.attn_norm.weight", "offset": 41388589056, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 41388621824, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.61.ffn_up.weight", "offset": 41570287616, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.61.ffn_norm.weight", "offset": 41751953408, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 41751986176, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 41752018944, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.62.ffn_gate.weight", "offset": 41933684736, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.62.ffn_up.weight", "offset": 42115350528, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.62.ffn_norm.weight", "offset": 42297016320, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 42297049088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 42297053184, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.62.attn_output.weight", "offset": 42303344640, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.62.attn_q.bias", "offset": 42353676288, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 42353709056, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.62.attn_v.bias", "offset": 42404040704, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 42404044800, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.63.ffn_gate.weight", "offset": 42410336256, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.63.ffn_up.weight", "offset": 42592002048, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.63.attn_k.bias", "offset": 42773667840, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 42773671936, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.63.attn_output.weight", "offset": 42779963392, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.63.attn_q.bias", "offset": 42830295040, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 42830327808, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.63.attn_v.bias", "offset": 42880659456, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 42880663552, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.63.attn_norm.weight", "offset": 42886955008, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 42886987776, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.63.ffn_norm.weight", "offset": 43068653568, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 43068686336, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 43068719104, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.64.ffn_gate.weight", "offset": 43250384896, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.64.ffn_up.weight", "offset": 43432050688, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.64.ffn_norm.weight", "offset": 43613716480, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 43613749248, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 43613753344, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.64.attn_output.weight", "offset": 43620044800, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.64.attn_q.bias", "offset": 43670376448, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 43670409216, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.64.attn_v.bias", "offset": 43720740864, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 43720744960, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.65.attn_norm.weight", "offset": 43727036416, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 43727069184, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.65.ffn_gate.weight", "offset": 43908734976, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.65.ffn_up.weight", "offset": 44090400768, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.65.ffn_norm.weight", "offset": 44272066560, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 44272099328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 44272103424, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.65.attn_output.weight", "offset": 44278394880, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.65.attn_q.bias", "offset": 44328726528, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 44328759296, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.65.attn_v.bias", "offset": 44379090944, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 44379095040, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.66.attn_norm.weight", "offset": 44385386496, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 44385419264, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.66.ffn_gate.weight", "offset": 44567085056, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.66.ffn_up.weight", "offset": 44748750848, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.66.ffn_norm.weight", "offset": 44930416640, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 44930449408, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 44930453504, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.66.attn_output.weight", "offset": 44936744960, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.66.attn_q.bias", "offset": 44987076608, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 44987109376, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.66.attn_v.bias", "offset": 45037441024, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 45037445120, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.67.attn_norm.weight", "offset": 45043736576, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 45043769344, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.67.ffn_gate.weight", "offset": 45225435136, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.67.ffn_up.weight", "offset": 45407100928, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.67.ffn_norm.weight", "offset": 45588766720, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 45588799488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 45588803584, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.67.attn_output.weight", "offset": 45595095040, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.67.attn_q.bias", "offset": 45645426688, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 45645459456, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.67.attn_v.bias", "offset": 45695791104, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 45695795200, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.68.attn_k.bias", "offset": 45702086656, "shape": [ 1024 ], "size": 6291456, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 45702090752, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.68.attn_output.weight", "offset": 45708382208, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.68.attn_q.bias", "offset": 45758713856, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 45758746624, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.68.attn_v.bias", "offset": 45809078272, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 45809082368, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.68.attn_norm.weight", "offset": 45815373824, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 45815406592, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.68.ffn_gate.weight", "offset": 45997072384, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.68.ffn_up.weight", "offset": 46178738176, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.68.ffn_norm.weight", "offset": 46360403968, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 46360436736, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 46360469504, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.69.ffn_gate.weight", "offset": 46542135296, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.69.ffn_up.weight", "offset": 46723801088, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.69.ffn_norm.weight", "offset": 46905466880, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 46905499648, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 46905503744, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.69.attn_output.weight", "offset": 46911795200, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.69.attn_q.bias", "offset": 46962126848, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 46962159616, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.69.attn_v.bias", "offset": 47012491264, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 47012495360, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.70.ffn_gate.weight", "offset": 47018786816, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.70.attn_k.bias", "offset": 47200452608, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 47200456704, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.70.attn_output.weight", "offset": 47206748160, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.70.attn_q.bias", "offset": 47257079808, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 47257112576, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.70.attn_v.bias", "offset": 47307444224, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 47307448320, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.70.attn_norm.weight", "offset": 47313739776, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 47313772544, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.70.ffn_up.weight", "offset": 47495438336, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.70.ffn_norm.weight", "offset": 47677104128, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 47677136896, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 47677169664, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.71.ffn_gate.weight", "offset": 47858835456, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.71.ffn_up.weight", "offset": 48040501248, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.71.ffn_norm.weight", "offset": 48222167040, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 48222199808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 48222203904, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.71.attn_output.weight", "offset": 48228495360, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.71.attn_q.bias", "offset": 48278827008, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 48278859776, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.71.attn_v.bias", "offset": 48329191424, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 48329195520, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.72.ffn_gate.weight", "offset": 48335486976, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.72.ffn_up.weight", "offset": 48517152768, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.72.attn_k.bias", "offset": 48698818560, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 48698822656, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.72.attn_output.weight", "offset": 48705114112, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.72.attn_q.bias", "offset": 48755445760, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 48755478528, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.72.attn_v.bias", "offset": 48805810176, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 48805814272, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.72.attn_norm.weight", "offset": 48812105728, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 48812138496, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.72.ffn_norm.weight", "offset": 48993804288, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 48993837056, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 48993869824, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.73.ffn_gate.weight", "offset": 49175535616, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.73.ffn_up.weight", "offset": 49357201408, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.73.ffn_norm.weight", "offset": 49538867200, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 49538899968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 49538904064, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.73.attn_output.weight", "offset": 49545195520, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.73.attn_q.bias", "offset": 49595527168, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 49595559936, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.73.attn_v.bias", "offset": 49645891584, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 49645895680, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.74.attn_norm.weight", "offset": 49652187136, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 49652219904, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.74.ffn_gate.weight", "offset": 49833885696, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.74.ffn_up.weight", "offset": 50015551488, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.74.ffn_norm.weight", "offset": 50197217280, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 50197250048, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 50197254144, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.74.attn_output.weight", "offset": 50203545600, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.74.attn_q.bias", "offset": 50253877248, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 50253910016, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.74.attn_v.bias", "offset": 50304241664, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 50304245760, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.75.attn_norm.weight", "offset": 50310537216, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 50310569984, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.75.ffn_gate.weight", "offset": 50492235776, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.75.ffn_up.weight", "offset": 50673901568, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.75.ffn_norm.weight", "offset": 50855567360, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 50855600128, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 50855604224, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.75.attn_output.weight", "offset": 50861895680, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.75.attn_q.bias", "offset": 50912227328, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 50912260096, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.75.attn_v.bias", "offset": 50962591744, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 50962595840, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.76.attn_norm.weight", "offset": 50968887296, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 50968920064, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.76.ffn_gate.weight", "offset": 51150585856, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.76.ffn_up.weight", "offset": 51332251648, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.76.ffn_norm.weight", "offset": 51513917440, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 51513950208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 51513954304, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.76.attn_output.weight", "offset": 51520245760, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.76.attn_q.bias", "offset": 51570577408, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 51570610176, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.76.attn_v.bias", "offset": 51620941824, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 51620945920, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.77.attn_k.bias", "offset": 51627237376, "shape": [ 1024 ], "size": 6291456, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 51627241472, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.77.attn_output.weight", "offset": 51633532928, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.77.attn_q.bias", "offset": 51683864576, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 51683897344, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.77.attn_v.bias", "offset": 51734228992, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 51734233088, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.77.attn_norm.weight", "offset": 51740524544, "shape": [ 8192 ], "size": 6291456, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 51740557312, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.77.ffn_gate.weight", "offset": 51922223104, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.77.ffn_up.weight", "offset": 52103888896, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.77.ffn_norm.weight", "offset": 52285554688, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 52285587456, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 52285620224, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.78.ffn_gate.weight", "offset": 52467286016, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.78.ffn_up.weight", "offset": 52648951808, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.78.ffn_norm.weight", "offset": 52830617600, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 52830650368, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 52830654464, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.78.attn_output.weight", "offset": 52836945920, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.78.attn_q.bias", "offset": 52887277568, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 52887310336, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.78.attn_v.bias", "offset": 52937641984, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 52937646080, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.79.ffn_gate.weight", "offset": 52943937536, "shape": [ 8192, 29568 ], "size": 6291456, "type": 7 }, { "name": "blk.79.attn_k.bias", "offset": 53125603328, "shape": [ 1024 ], "size": 181665792, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 53125607424, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "blk.79.attn_output.weight", "offset": 53131898880, "shape": [ 8192, 8192 ], "size": 6291456, "type": 7 }, { "name": "blk.79.attn_q.bias", "offset": 53182230528, "shape": [ 8192 ], "size": 50331648, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 53182263296, "shape": [ 8192, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.79.attn_v.bias", "offset": 53232594944, "shape": [ 1024 ], "size": 50331648, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 53232599040, "shape": [ 8192, 1024 ], "size": 4096, "type": 7 }, { "name": "output.weight", "offset": 53238890496, "shape": [ 8192, 152064 ], "size": 6291456, "type": 14 }, { "name": "blk.79.attn_norm.weight", "offset": 54260760576, "shape": [ 8192 ], "size": 1021870080, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 54260793344, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.79.ffn_up.weight", "offset": 54442459136, "shape": [ 8192, 29568 ], "size": 181665792, "type": 7 }, { "name": "blk.79.ffn_norm.weight", "offset": 54624124928, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "output_norm.weight", "offset": 54624157696, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }