qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 16, "general.name": "Qwen2-72B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 80, "qwen2.context_length": 32768, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 856424448, "shape": [ 8192, 29568 ], "size": 856424448, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 1022951424, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.0.attn_k.bias", "offset": 1189478400, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 1189482496, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 1195249664, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.0.attn_q.bias", "offset": 1241387008, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 1241419776, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.0.attn_v.bias", "offset": 1287557120, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 1287561216, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 1293328384, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 1293361152, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 1475026944, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1475059712, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1475092480, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.1.ffn_gate.weight", "offset": 1656758272, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 1823285248, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 1989812224, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 1989844992, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1989849088, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 1995616256, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.1.attn_q.bias", "offset": 2041753600, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 2041786368, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.1.attn_v.bias", "offset": 2087923712, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 2087927808, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 2093694976, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 2093727744, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.2.ffn_gate.weight", "offset": 2275393536, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 2441920512, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 2608447488, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 2608480256, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 2608484352, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 2614251520, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.2.attn_q.bias", "offset": 2660388864, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 2660421632, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.2.attn_v.bias", "offset": 2706558976, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 2706563072, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 2712330240, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 2712363008, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.3.ffn_gate.weight", "offset": 2894028800, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 3060555776, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 3227082752, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 3227115520, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 3227119616, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 3232886784, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.3.attn_q.bias", "offset": 3279024128, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 3279056896, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.3.attn_v.bias", "offset": 3325194240, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 3325198336, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 3330965504, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 3330998272, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.4.ffn_gate.weight", "offset": 3512664064, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 3679191040, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 3845718016, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 3845750784, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 3845754880, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 3851522048, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.4.attn_q.bias", "offset": 3897659392, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 3897692160, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.4.attn_v.bias", "offset": 3943829504, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 3943833600, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_k.bias", "offset": 3949600768, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 3949604864, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 3955372032, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.5.attn_q.bias", "offset": 4001509376, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 4001542144, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.5.attn_v.bias", "offset": 4047679488, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 4047683584, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 4053450752, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 4053483520, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.5.ffn_gate.weight", "offset": 4235149312, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 4401676288, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 4568203264, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 4568236032, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 4568268800, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.6.ffn_gate.weight", "offset": 4749934592, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 4916461568, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 5082988544, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 5083021312, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 5083025408, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 5088792576, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.6.attn_q.bias", "offset": 5134929920, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 5134962688, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.6.attn_v.bias", "offset": 5181100032, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 5181104128, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 5186871296, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.7.attn_k.bias", "offset": 5353398272, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 5353402368, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 5359169536, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.7.attn_q.bias", "offset": 5405306880, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 5405339648, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.7.attn_v.bias", "offset": 5451476992, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 5451481088, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 5457248256, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 5457281024, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 5638946816, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 5805473792, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 5805506560, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 5805539328, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.8.ffn_gate.weight", "offset": 5987205120, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 6153732096, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 6320259072, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 6320291840, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 6320295936, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 6326063104, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.8.attn_q.bias", "offset": 6372200448, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 6372233216, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.8.attn_v.bias", "offset": 6418370560, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 6418374656, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 6424141824, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.9.ffn_up.weight", "offset": 6590668800, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.9.attn_k.bias", "offset": 6757195776, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 6757199872, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 6762967040, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.9.attn_q.bias", "offset": 6809104384, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 6809137152, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.9.attn_v.bias", "offset": 6855274496, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 6855278592, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 6861045760, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 6861078528, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.10.ffn_gate.weight", "offset": 7042744320, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.10.ffn_up.weight", "offset": 7209271296, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 7375798272, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 7375831040, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 7375835136, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 7381602304, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.10.attn_q.bias", "offset": 7427739648, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 7427772416, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.10.attn_v.bias", "offset": 7473909760, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 7473913856, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 7479681024, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 7479713792, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.11.ffn_gate.weight", "offset": 7661379584, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.11.ffn_up.weight", "offset": 7827906560, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 7994433536, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 7994466304, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 7994470400, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 8000237568, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.11.attn_q.bias", "offset": 8046374912, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 8046407680, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.11.attn_v.bias", "offset": 8092545024, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 8092549120, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 8098316288, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 8098349056, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 8280014848, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 8280047616, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 8280080384, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.12.ffn_gate.weight", "offset": 8461746176, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.12.ffn_up.weight", "offset": 8628273152, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 8794800128, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 8794832896, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 8794836992, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 8800604160, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.12.attn_q.bias", "offset": 8846741504, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 8846774272, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.12.attn_v.bias", "offset": 8892911616, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 8892915712, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 8898682880, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 8898715648, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.13.ffn_gate.weight", "offset": 9080381440, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.13.ffn_up.weight", "offset": 9246908416, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 9413435392, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 9413468160, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 9413472256, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 9419239424, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.13.attn_q.bias", "offset": 9465376768, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 9465409536, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.13.attn_v.bias", "offset": 9511546880, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 9511550976, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_k.bias", "offset": 9517318144, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 9517322240, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 9523089408, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.14.attn_q.bias", "offset": 9569226752, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 9569259520, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.14.attn_v.bias", "offset": 9615396864, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 9615400960, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 9621168128, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 9621200896, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.14.ffn_gate.weight", "offset": 9802866688, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 9969393664, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 10135920640, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 10135953408, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 10135986176, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.15.ffn_gate.weight", "offset": 10317651968, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 10484178944, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 10650705920, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 10650738688, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 10650742784, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 10656509952, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.15.attn_q.bias", "offset": 10702647296, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 10702680064, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.15.attn_v.bias", "offset": 10748817408, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 10748821504, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 10754588672, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.16.attn_k.bias", "offset": 10921115648, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 10921119744, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 10926886912, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.16.attn_q.bias", "offset": 10973024256, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 10973057024, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.16.attn_v.bias", "offset": 11019194368, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 11019198464, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 11024965632, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 11024998400, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 11206664192, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000P˂\u0003\u0000\u0000\u0000\u0014\u0000\u0000\u0000\u0000", "offset": 11373191168, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 11373223936, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 11373256704, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.17.ffn_gate.weight", "offset": 11554922496, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.17.ffn_up.weight", "offset": 11721449472, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 11887976448, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 11888009216, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 11888013312, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 11893780480, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.17.attn_q.bias", "offset": 11939917824, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 11939950592, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.17.attn_v.bias", "offset": 11986087936, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 11986092032, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 11991859200, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 12158386176, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.18.attn_k.bias", "offset": 12324913152, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 12324917248, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 12330684416, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.18.attn_q.bias", "offset": 12376821760, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 12376854528, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.18.attn_v.bias", "offset": 12422991872, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 12422995968, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 12428763136, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 12428795904, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 12610461696, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 12610494464, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 12610527232, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.19.ffn_gate.weight", "offset": 12792193024, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.19.ffn_up.weight", "offset": 12958720000, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 13125246976, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 13125279744, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 13125283840, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 13131051008, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.19.attn_q.bias", "offset": 13177188352, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 13177221120, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.19.attn_v.bias", "offset": 13223358464, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 13223362560, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 13229129728, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 13229162496, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.20.ffn_gate.weight", "offset": 13410828288, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 13577355264, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 13743882240, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 13743915008, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 13743919104, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 13749686272, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.20.attn_q.bias", "offset": 13795823616, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 13795856384, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.20.attn_v.bias", "offset": 13841993728, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 13841997824, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 13847764992, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 13847797760, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.21.ffn_gate.weight", "offset": 14029463552, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 14195990528, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 14362517504, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 14362550272, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 14362554368, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 14368321536, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.21.attn_q.bias", "offset": 14414458880, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 14414491648, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.21.attn_v.bias", "offset": 14460628992, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 14460633088, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 14466400256, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 14466433024, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.22.ffn_gate.weight", "offset": 14648098816, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.22.ffn_up.weight", "offset": 14814625792, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 14981152768, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 14981185536, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 14981189632, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 14986956800, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.22.attn_q.bias", "offset": 15033094144, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 15033126912, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.22.attn_v.bias", "offset": 15079264256, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 15079268352, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_k.bias", "offset": 15085035520, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 15085039616, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 15090806784, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.23.attn_q.bias", "offset": 15136944128, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 15136976896, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.23.attn_v.bias", "offset": 15183114240, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 15183118336, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 15188885504, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 15188918272, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.23.ffn_gate.weight", "offset": 15370584064, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 15537111040, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 15703638016, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 15703670784, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 15703703552, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.24.ffn_gate.weight", "offset": 15885369344, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 16051896320, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 16218423296, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 16218456064, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 16218460160, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 16224227328, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.24.attn_q.bias", "offset": 16270364672, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 16270397440, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.24.attn_v.bias", "offset": 16316534784, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 16316538880, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 16322306048, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.25.attn_k.bias", "offset": 16488833024, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 16488837120, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 16494604288, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.25.attn_q.bias", "offset": 16540741632, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 16540774400, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.25.attn_v.bias", "offset": 16586911744, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 16586915840, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 16592683008, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 16592715776, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.25.ffn_up.weight", "offset": 16774381568, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 16940908544, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 16940941312, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 16940974080, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.26.ffn_gate.weight", "offset": 17122639872, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 17289166848, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 17455693824, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 17455726592, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 17455730688, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 17461497856, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.26.attn_q.bias", "offset": 17507635200, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 17507667968, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.26.attn_v.bias", "offset": 17553805312, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 17553809408, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 17559576576, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 17726103552, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.27.attn_k.bias", "offset": 17892630528, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 17892634624, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.attn_output.weight", "offset": 17898401792, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.27.attn_q.bias", "offset": 17944539136, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 17944571904, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.27.attn_v.bias", "offset": 17990709248, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 17990713344, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 17996480512, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 17996513280, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.27.ffn_norm.weight", "offset": 18178179072, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 18178211840, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 18178244608, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.28.ffn_gate.weight", "offset": 18359910400, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.28.ffn_up.weight", "offset": 18526437376, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.28.ffn_norm.weight", "offset": 18692964352, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 18692997120, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 18693001216, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.28.attn_output.weight", "offset": 18698768384, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.28.attn_q.bias", "offset": 18744905728, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 18744938496, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.28.attn_v.bias", "offset": 18791075840, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 18791079936, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 18796847104, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 18796879872, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.29.ffn_gate.weight", "offset": 18978545664, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.29.ffn_up.weight", "offset": 19145072640, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.29.ffn_norm.weight", "offset": 19311599616, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 19311632384, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 19311636480, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.29.attn_output.weight", "offset": 19317403648, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.29.attn_q.bias", "offset": 19363540992, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 19363573760, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.29.attn_v.bias", "offset": 19409711104, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 19409715200, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 19415482368, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 19415515136, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.30.ffn_gate.weight", "offset": 19597180928, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.30.ffn_up.weight", "offset": 19763707904, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.30.ffn_norm.weight", "offset": 19930234880, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 19930267648, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 19930271744, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.30.attn_output.weight", "offset": 19936038912, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.30.attn_q.bias", "offset": 19982176256, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 19982209024, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.30.attn_v.bias", "offset": 20028346368, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 20028350464, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 20034117632, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 20034150400, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.31.ffn_gate.weight", "offset": 20215816192, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.31.ffn_up.weight", "offset": 20382343168, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.31.ffn_norm.weight", "offset": 20548870144, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 20548902912, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 20548907008, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.31.attn_output.weight", "offset": 20554674176, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.31.attn_q.bias", "offset": 20600811520, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 20600844288, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.31.attn_v.bias", "offset": 20646981632, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 20646985728, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_k.bias", "offset": 20652752896, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 20652756992, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_output.weight", "offset": 20658524160, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.32.attn_q.bias", "offset": 20704661504, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 20704694272, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.32.attn_v.bias", "offset": 20750831616, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 20750835712, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 20756602880, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 20756635648, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.32.ffn_gate.weight", "offset": 20938301440, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.32.ffn_up.weight", "offset": 21104828416, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.32.ffn_norm.weight", "offset": 21271355392, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 21271388160, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 21271420928, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.33.ffn_gate.weight", "offset": 21453086720, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.33.ffn_up.weight", "offset": 21619613696, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.33.ffn_norm.weight", "offset": 21786140672, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 21786173440, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 21786177536, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.33.attn_output.weight", "offset": 21791944704, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.33.attn_q.bias", "offset": 21838082048, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 21838114816, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.33.attn_v.bias", "offset": 21884252160, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 21884256256, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.ffn_gate.weight", "offset": 21890023424, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.34.attn_k.bias", "offset": 22056550400, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 22056554496, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.attn_output.weight", "offset": 22062321664, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.34.attn_q.bias", "offset": 22108459008, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 22108491776, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.34.attn_v.bias", "offset": 22154629120, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 22154633216, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 22160400384, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 22160433152, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.34.ffn_up.weight", "offset": 22342098944, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.34.ffn_norm.weight", "offset": 22508625920, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 22508658688, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 22508691456, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.35.ffn_gate.weight", "offset": 22690357248, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.35.ffn_up.weight", "offset": 22856884224, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.35.ffn_norm.weight", "offset": 23023411200, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 23023443968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 23023448064, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.35.attn_output.weight", "offset": 23029215232, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.35.attn_q.bias", "offset": 23075352576, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 23075385344, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.35.attn_v.bias", "offset": 23121522688, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 23121526784, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 23127293952, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.36.ffn_up.weight", "offset": 23293820928, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.36.attn_k.bias", "offset": 23460347904, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 23460352000, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.attn_output.weight", "offset": 23466119168, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.36.attn_q.bias", "offset": 23512256512, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 23512289280, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.36.attn_v.bias", "offset": 23558426624, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 23558430720, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 23564197888, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 23564230656, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.36.ffn_norm.weight", "offset": 23745896448, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 23745929216, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 23745961984, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.37.ffn_gate.weight", "offset": 23927627776, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.37.ffn_up.weight", "offset": 24094154752, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.37.ffn_norm.weight", "offset": 24260681728, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 24260714496, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 24260718592, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.37.attn_output.weight", "offset": 24266485760, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.37.attn_q.bias", "offset": 24312623104, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 24312655872, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.37.attn_v.bias", "offset": 24358793216, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 24358797312, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 24364564480, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 24364597248, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.38.ffn_gate.weight", "offset": 24546263040, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.38.ffn_up.weight", "offset": 24712790016, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.38.ffn_norm.weight", "offset": 24879316992, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 24879349760, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 24879353856, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.38.attn_output.weight", "offset": 24885121024, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.38.attn_q.bias", "offset": 24931258368, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 24931291136, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.38.attn_v.bias", "offset": 24977428480, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 24977432576, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 24983199744, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 24983232512, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.39.ffn_gate.weight", "offset": 25164898304, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.39.ffn_up.weight", "offset": 25331425280, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.39.ffn_norm.weight", "offset": 25497952256, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 25497985024, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 25497989120, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.39.attn_output.weight", "offset": 25503756288, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.39.attn_q.bias", "offset": 25549893632, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 25549926400, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.39.attn_v.bias", "offset": 25596063744, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 25596067840, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 25601835008, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 25601867776, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.40.ffn_gate.weight", "offset": 25783533568, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.40.ffn_up.weight", "offset": 25950060544, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.40.ffn_norm.weight", "offset": 26116587520, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 26116620288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 26116624384, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.40.attn_output.weight", "offset": 26122391552, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.40.attn_q.bias", "offset": 26168528896, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 26168561664, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.40.attn_v.bias", "offset": 26214699008, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 26214703104, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_k.bias", "offset": 26220470272, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 26220474368, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_output.weight", "offset": 26226241536, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.41.attn_q.bias", "offset": 26272378880, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 26272411648, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.41.attn_v.bias", "offset": 26318548992, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 26318553088, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 26324320256, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 26324353024, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.41.ffn_gate.weight", "offset": 26506018816, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.41.ffn_up.weight", "offset": 26672545792, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.41.ffn_norm.weight", "offset": 26839072768, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 26839105536, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 26839138304, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.42.ffn_gate.weight", "offset": 27020804096, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.42.ffn_up.weight", "offset": 27187331072, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.42.ffn_norm.weight", "offset": 27353858048, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 27353890816, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 27353894912, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.42.attn_output.weight", "offset": 27359662080, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.42.attn_q.bias", "offset": 27405799424, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 27405832192, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.42.attn_v.bias", "offset": 27451969536, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 27451973632, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.ffn_gate.weight", "offset": 27457740800, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.43.attn_k.bias", "offset": 27624267776, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 27624271872, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.attn_output.weight", "offset": 27630039040, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.43.attn_q.bias", "offset": 27676176384, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 27676209152, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.43.attn_v.bias", "offset": 27722346496, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 27722350592, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 27728117760, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 27728150528, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.43.ffn_up.weight", "offset": 27909816320, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.43.ffn_norm.weight", "offset": 28076343296, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 28076376064, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 28076408832, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.44.ffn_gate.weight", "offset": 28258074624, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.44.ffn_up.weight", "offset": 28424601600, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.44.ffn_norm.weight", "offset": 28591128576, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 28591161344, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 28591165440, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.44.attn_output.weight", "offset": 28596932608, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.44.attn_q.bias", "offset": 28643069952, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 28643102720, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.44.attn_v.bias", "offset": 28689240064, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 28689244160, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.ffn_gate.weight", "offset": 28695011328, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.45.ffn_up.weight", "offset": 28861538304, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.45.attn_k.bias", "offset": 29028065280, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 29028069376, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.attn_output.weight", "offset": 29033836544, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.45.attn_q.bias", "offset": 29079973888, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 29080006656, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.45.attn_v.bias", "offset": 29126144000, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 29126148096, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 29131915264, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 29131948032, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.45.ffn_norm.weight", "offset": 29313613824, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 29313646592, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 29313679360, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.46.ffn_gate.weight", "offset": 29495345152, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.46.ffn_up.weight", "offset": 29661872128, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.46.ffn_norm.weight", "offset": 29828399104, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 29828431872, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 29828435968, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.46.attn_output.weight", "offset": 29834203136, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.46.attn_q.bias", "offset": 29880340480, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 29880373248, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.46.attn_v.bias", "offset": 29926510592, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 29926514688, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 29932281856, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 29932314624, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.47.ffn_gate.weight", "offset": 30113980416, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.47.ffn_up.weight", "offset": 30280507392, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.47.ffn_norm.weight", "offset": 30447034368, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 30447067136, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 30447071232, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.47.attn_output.weight", "offset": 30452838400, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.47.attn_q.bias", "offset": 30498975744, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 30499008512, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.47.attn_v.bias", "offset": 30545145856, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 30545149952, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.48.attn_norm.weight", "offset": 30550917120, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 30550949888, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.48.ffn_gate.weight", "offset": 30732615680, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.48.ffn_up.weight", "offset": 30899142656, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.48.ffn_norm.weight", "offset": 31065669632, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 31065702400, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 31065706496, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.48.attn_output.weight", "offset": 31071473664, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.48.attn_q.bias", "offset": 31117611008, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 31117643776, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.48.attn_v.bias", "offset": 31163781120, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 31163785216, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.49.attn_norm.weight", "offset": 31169552384, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 31169585152, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.49.ffn_gate.weight", "offset": 31351250944, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.49.ffn_up.weight", "offset": 31517777920, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.49.ffn_norm.weight", "offset": 31684304896, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 31684337664, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 31684341760, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.49.attn_output.weight", "offset": 31690108928, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.49.attn_q.bias", "offset": 31736246272, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 31736279040, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.49.attn_v.bias", "offset": 31782416384, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 31782420480, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_k.bias", "offset": 31788187648, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 31788191744, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_output.weight", "offset": 31793958912, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.50.attn_q.bias", "offset": 31840096256, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 31840129024, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.50.attn_v.bias", "offset": 31886266368, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 31886270464, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_norm.weight", "offset": 31892037632, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 31892070400, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.50.ffn_gate.weight", "offset": 32073736192, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.50.ffn_up.weight", "offset": 32240263168, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.50.ffn_norm.weight", "offset": 32406790144, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 32406822912, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 32406855680, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.51.ffn_gate.weight", "offset": 32588521472, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.51.ffn_up.weight", "offset": 32755048448, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.51.ffn_norm.weight", "offset": 32921575424, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 32921608192, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 32921612288, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.51.attn_output.weight", "offset": 32927379456, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.51.attn_q.bias", "offset": 32973516800, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 32973549568, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.51.attn_v.bias", "offset": 33019686912, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 33019691008, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.ffn_gate.weight", "offset": 33025458176, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.52.attn_k.bias", "offset": 33191985152, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 33191989248, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.attn_output.weight", "offset": 33197756416, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.52.attn_q.bias", "offset": 33243893760, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 33243926528, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.52.attn_v.bias", "offset": 33290063872, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 33290067968, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.attn_norm.weight", "offset": 33295835136, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 33295867904, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.52.ffn_up.weight", "offset": 33477533696, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.52.ffn_norm.weight", "offset": 33644060672, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 33644093440, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 33644126208, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.53.ffn_gate.weight", "offset": 33825792000, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.53.ffn_up.weight", "offset": 33992318976, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.53.ffn_norm.weight", "offset": 34158845952, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 34158878720, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 34158882816, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.53.attn_output.weight", "offset": 34164649984, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.53.attn_q.bias", "offset": 34210787328, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 34210820096, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.53.attn_v.bias", "offset": 34256957440, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 34256961536, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.ffn_gate.weight", "offset": 34262728704, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.54.ffn_up.weight", "offset": 34429255680, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.54.attn_k.bias", "offset": 34595782656, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 34595786752, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.attn_output.weight", "offset": 34601553920, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.54.attn_q.bias", "offset": 34647691264, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 34647724032, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.54.attn_v.bias", "offset": 34693861376, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 34693865472, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.attn_norm.weight", "offset": 34699632640, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 34699665408, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.54.ffn_norm.weight", "offset": 34881331200, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 34881363968, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 34881396736, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.55.ffn_gate.weight", "offset": 35063062528, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.55.ffn_up.weight", "offset": 35229589504, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.55.ffn_norm.weight", "offset": 35396116480, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 35396149248, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 35396153344, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.55.attn_output.weight", "offset": 35401920512, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.55.attn_q.bias", "offset": 35448057856, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 35448090624, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.55.attn_v.bias", "offset": 35494227968, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 35494232064, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.56.attn_norm.weight", "offset": 35499999232, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 35500032000, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.56.ffn_gate.weight", "offset": 35681697792, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.56.ffn_up.weight", "offset": 35848224768, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.56.ffn_norm.weight", "offset": 36014751744, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 36014784512, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 36014788608, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.56.attn_output.weight", "offset": 36020555776, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.56.attn_q.bias", "offset": 36066693120, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 36066725888, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.56.attn_v.bias", "offset": 36112863232, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 36112867328, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.57.attn_norm.weight", "offset": 36118634496, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 36118667264, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.57.ffn_gate.weight", "offset": 36300333056, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.57.ffn_up.weight", "offset": 36466860032, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.57.ffn_norm.weight", "offset": 36633387008, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 36633419776, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 36633423872, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.57.attn_output.weight", "offset": 36639191040, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.57.attn_q.bias", "offset": 36685328384, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 36685361152, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.57.attn_v.bias", "offset": 36731498496, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 36731502592, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.58.attn_norm.weight", "offset": 36737269760, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 36737302528, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.58.ffn_gate.weight", "offset": 36918968320, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.58.ffn_up.weight", "offset": 37085495296, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.58.ffn_norm.weight", "offset": 37252022272, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 37252055040, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 37252059136, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.58.attn_output.weight", "offset": 37257826304, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.58.attn_q.bias", "offset": 37303963648, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 37303996416, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.58.attn_v.bias", "offset": 37350133760, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 37350137856, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_k.bias", "offset": 37355905024, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 37355909120, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_output.weight", "offset": 37361676288, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.59.attn_q.bias", "offset": 37407813632, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 37407846400, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.59.attn_v.bias", "offset": 37453983744, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 37453987840, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_norm.weight", "offset": 37459755008, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 37459787776, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.59.ffn_gate.weight", "offset": 37641453568, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.59.ffn_up.weight", "offset": 37807980544, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.59.ffn_norm.weight", "offset": 37974507520, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 37974540288, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 37974573056, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.60.ffn_gate.weight", "offset": 38156238848, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.60.ffn_up.weight", "offset": 38322765824, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.60.ffn_norm.weight", "offset": 38489292800, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 38489325568, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 38489329664, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.60.attn_output.weight", "offset": 38495096832, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.60.attn_q.bias", "offset": 38541234176, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 38541266944, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.60.attn_v.bias", "offset": 38587404288, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 38587408384, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.ffn_gate.weight", "offset": 38593175552, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.61.attn_k.bias", "offset": 38759702528, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 38759706624, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.attn_output.weight", "offset": 38765473792, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.61.attn_q.bias", "offset": 38811611136, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 38811643904, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.61.attn_v.bias", "offset": 38857781248, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 38857785344, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.attn_norm.weight", "offset": 38863552512, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 38863585280, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.61.ffn_up.weight", "offset": 39045251072, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.61.ffn_norm.weight", "offset": 39211778048, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 39211810816, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 39211843584, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.62.ffn_gate.weight", "offset": 39393509376, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.62.ffn_up.weight", "offset": 39560036352, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.62.ffn_norm.weight", "offset": 39726563328, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 39726596096, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 39726600192, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.62.attn_output.weight", "offset": 39732367360, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.62.attn_q.bias", "offset": 39778504704, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 39778537472, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.62.attn_v.bias", "offset": 39824674816, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 39824678912, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.ffn_gate.weight", "offset": 39830446080, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.63.ffn_up.weight", "offset": 39996973056, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.63.attn_k.bias", "offset": 40163500032, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 40163504128, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.attn_output.weight", "offset": 40169271296, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.63.attn_q.bias", "offset": 40215408640, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 40215441408, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.63.attn_v.bias", "offset": 40261578752, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 40261582848, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.attn_norm.weight", "offset": 40267350016, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 40267382784, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.63.ffn_norm.weight", "offset": 40449048576, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 40449081344, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 40449114112, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.64.ffn_gate.weight", "offset": 40630779904, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.64.ffn_up.weight", "offset": 40797306880, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.64.ffn_norm.weight", "offset": 40963833856, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 40963866624, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 40963870720, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.64.attn_output.weight", "offset": 40969637888, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.64.attn_q.bias", "offset": 41015775232, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 41015808000, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.64.attn_v.bias", "offset": 41061945344, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 41061949440, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.65.attn_norm.weight", "offset": 41067716608, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 41067749376, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.65.ffn_gate.weight", "offset": 41249415168, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.65.ffn_up.weight", "offset": 41415942144, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.65.ffn_norm.weight", "offset": 41582469120, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 41582501888, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 41582505984, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.65.attn_output.weight", "offset": 41588273152, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.65.attn_q.bias", "offset": 41634410496, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 41634443264, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.65.attn_v.bias", "offset": 41680580608, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 41680584704, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.66.attn_norm.weight", "offset": 41686351872, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 41686384640, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.66.ffn_gate.weight", "offset": 41868050432, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.66.ffn_up.weight", "offset": 42034577408, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.66.ffn_norm.weight", "offset": 42201104384, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 42201137152, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 42201141248, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.66.attn_output.weight", "offset": 42206908416, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.66.attn_q.bias", "offset": 42253045760, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 42253078528, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.66.attn_v.bias", "offset": 42299215872, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 42299219968, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.67.attn_norm.weight", "offset": 42304987136, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 42305019904, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.67.ffn_gate.weight", "offset": 42486685696, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.67.ffn_up.weight", "offset": 42653212672, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.67.ffn_norm.weight", "offset": 42819739648, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 42819772416, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 42819776512, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.67.attn_output.weight", "offset": 42825543680, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.67.attn_q.bias", "offset": 42871681024, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 42871713792, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.67.attn_v.bias", "offset": 42917851136, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 42917855232, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_k.bias", "offset": 42923622400, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 42923626496, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_output.weight", "offset": 42929393664, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.68.attn_q.bias", "offset": 42975531008, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 42975563776, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.68.attn_v.bias", "offset": 43021701120, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 43021705216, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_norm.weight", "offset": 43027472384, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 43027505152, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.68.ffn_gate.weight", "offset": 43209170944, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.68.ffn_up.weight", "offset": 43375697920, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.68.ffn_norm.weight", "offset": 43542224896, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 43542257664, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 43542290432, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.69.ffn_gate.weight", "offset": 43723956224, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.69.ffn_up.weight", "offset": 43890483200, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.69.ffn_norm.weight", "offset": 44057010176, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 44057042944, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 44057047040, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.69.attn_output.weight", "offset": 44062814208, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.69.attn_q.bias", "offset": 44108951552, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 44108984320, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.69.attn_v.bias", "offset": 44155121664, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 44155125760, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.ffn_gate.weight", "offset": 44160892928, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.70.attn_k.bias", "offset": 44327419904, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 44327424000, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.attn_output.weight", "offset": 44333191168, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.70.attn_q.bias", "offset": 44379328512, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 44379361280, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.70.attn_v.bias", "offset": 44425498624, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 44425502720, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.attn_norm.weight", "offset": 44431269888, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 44431302656, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.70.ffn_up.weight", "offset": 44612968448, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.70.ffn_norm.weight", "offset": 44779495424, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 44779528192, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 44779560960, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.71.ffn_gate.weight", "offset": 44961226752, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.71.ffn_up.weight", "offset": 45127753728, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.71.ffn_norm.weight", "offset": 45294280704, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 45294313472, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 45294317568, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.71.attn_output.weight", "offset": 45300084736, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.71.attn_q.bias", "offset": 45346222080, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 45346254848, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.71.attn_v.bias", "offset": 45392392192, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 45392396288, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.ffn_gate.weight", "offset": 45398163456, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.72.ffn_up.weight", "offset": 45564690432, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.72.attn_k.bias", "offset": 45731217408, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 45731221504, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.attn_output.weight", "offset": 45736988672, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.72.attn_q.bias", "offset": 45783126016, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 45783158784, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.72.attn_v.bias", "offset": 45829296128, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 45829300224, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.attn_norm.weight", "offset": 45835067392, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 45835100160, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.72.ffn_norm.weight", "offset": 46016765952, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 46016798720, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 46016831488, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.73.ffn_gate.weight", "offset": 46198497280, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.73.ffn_up.weight", "offset": 46365024256, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.73.ffn_norm.weight", "offset": 46531551232, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 46531584000, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 46531588096, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.73.attn_output.weight", "offset": 46537355264, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.73.attn_q.bias", "offset": 46583492608, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 46583525376, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.73.attn_v.bias", "offset": 46629662720, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 46629666816, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.74.attn_norm.weight", "offset": 46635433984, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 46635466752, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.74.ffn_gate.weight", "offset": 46817132544, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.74.ffn_up.weight", "offset": 46983659520, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.74.ffn_norm.weight", "offset": 47150186496, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 47150219264, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 47150223360, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.74.attn_output.weight", "offset": 47155990528, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.74.attn_q.bias", "offset": 47202127872, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 47202160640, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.74.attn_v.bias", "offset": 47248297984, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 47248302080, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.75.attn_norm.weight", "offset": 47254069248, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 47254102016, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.75.ffn_gate.weight", "offset": 47435767808, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.75.ffn_up.weight", "offset": 47602294784, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.75.ffn_norm.weight", "offset": 47768821760, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 47768854528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 47768858624, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.75.attn_output.weight", "offset": 47774625792, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.75.attn_q.bias", "offset": 47820763136, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 47820795904, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.75.attn_v.bias", "offset": 47866933248, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 47866937344, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.76.attn_norm.weight", "offset": 47872704512, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 47872737280, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.76.ffn_gate.weight", "offset": 48054403072, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.76.ffn_up.weight", "offset": 48220930048, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.76.ffn_norm.weight", "offset": 48387457024, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 48387489792, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 48387493888, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.76.attn_output.weight", "offset": 48393261056, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.76.attn_q.bias", "offset": 48439398400, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 48439431168, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.76.attn_v.bias", "offset": 48485568512, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 48485572608, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_k.bias", "offset": 48491339776, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 48491343872, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_output.weight", "offset": 48497111040, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.77.attn_q.bias", "offset": 48543248384, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 48543281152, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.77.attn_v.bias", "offset": 48589418496, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 48589422592, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_norm.weight", "offset": 48595189760, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 48595222528, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.77.ffn_gate.weight", "offset": 48776888320, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.77.ffn_up.weight", "offset": 48943415296, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.77.ffn_norm.weight", "offset": 49109942272, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 49109975040, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 49110007808, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.78.ffn_gate.weight", "offset": 49291673600, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.78.ffn_up.weight", "offset": 49458200576, "shape": [ 8192, 29568 ], "size": 166526976, "type": 13 }, { "name": "blk.78.ffn_norm.weight", "offset": 49624727552, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 49624760320, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 49624764416, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.78.attn_output.weight", "offset": 49630531584, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.78.attn_q.bias", "offset": 49676668928, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 49676701696, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.78.attn_v.bias", "offset": 49722839040, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 49722843136, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.79.ffn_gate.weight", "offset": 49728610304, "shape": [ 8192, 29568 ], "size": 5767168, "type": 13 }, { "name": "blk.79.attn_k.bias", "offset": 49895137280, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 49895141376, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.79.attn_output.weight", "offset": 49900908544, "shape": [ 8192, 8192 ], "size": 5767168, "type": 13 }, { "name": "blk.79.attn_q.bias", "offset": 49947045888, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 49947078656, "shape": [ 8192, 8192 ], "size": 32768, "type": 13 }, { "name": "blk.79.attn_v.bias", "offset": 49993216000, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 49993220096, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "output.weight", "offset": 49998987264, "shape": [ 8192, 152064 ], "size": 5767168, "type": 14 }, { "name": "blk.79.attn_norm.weight", "offset": 51020857344, "shape": [ 8192 ], "size": 1021870080, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 51020890112, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.79.ffn_up.weight", "offset": 51202555904, "shape": [ 8192, 29568 ], "size": 181665792, "type": 13 }, { "name": "blk.79.ffn_norm.weight", "offset": 51369082880, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "output_norm.weight", "offset": 51369115648, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }