qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 15, "general.name": "Qwen2-72B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 80, "qwen2.context_length": 32768, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 700710912, "shape": [ 8192, 29568 ], "size": 700710912, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 836960256, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.0.attn_k.bias", "offset": 973209600, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 973213696, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 977932288, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.0.attn_q.bias", "offset": 1015681024, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 1015713792, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.0.attn_v.bias", "offset": 1053462528, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 1053466624, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.0.attn_norm.weight", "offset": 1060347904, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 1060380672, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 1317740544, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1317773312, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1317806080, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.1.ffn_gate.weight", "offset": 1575165952, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 1711415296, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 1847664640, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 1847697408, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1847701504, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 1852420096, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.1.attn_q.bias", "offset": 1890168832, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 1890201600, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.1.attn_v.bias", "offset": 1927950336, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 1927954432, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 1934835712, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1934868480, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.2.ffn_gate.weight", "offset": 2192228352, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 2328477696, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 2464727040, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 2464759808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 2464763904, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 2469482496, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.2.attn_q.bias", "offset": 2507231232, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 2507264000, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.2.attn_v.bias", "offset": 2545012736, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 2545016832, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 2551898112, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 2551930880, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.3.ffn_gate.weight", "offset": 2809290752, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 2945540096, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 3081789440, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 3081822208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 3081826304, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 3086544896, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.3.attn_q.bias", "offset": 3124293632, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 3124326400, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.3.attn_v.bias", "offset": 3162075136, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 3162079232, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 3168960512, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 3168993280, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.4.ffn_gate.weight", "offset": 3426353152, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 3562602496, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 3698851840, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 3698884608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 3698888704, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 3703607296, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.4.attn_q.bias", "offset": 3741356032, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 3741388800, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.4.attn_v.bias", "offset": 3779137536, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 3779141632, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.5.attn_k.bias", "offset": 3786022912, "shape": [ 1024 ], "size": 6881280, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 3786027008, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 3790745600, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.5.attn_q.bias", "offset": 3828494336, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 3828527104, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.5.attn_v.bias", "offset": 3866275840, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 3866279936, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 3873161216, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 3873193984, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.5.ffn_gate.weight", "offset": 4130553856, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 4266803200, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 4403052544, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 4403085312, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 4403118080, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.6.ffn_gate.weight", "offset": 4660477952, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 4796727296, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 4932976640, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 4933009408, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 4933013504, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 4937732096, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.6.attn_q.bias", "offset": 4975480832, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 4975513600, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.6.attn_v.bias", "offset": 5013262336, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 5013266432, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 5020147712, "shape": [ 8192, 29568 ], "size": 6881280, "type": 12 }, { "name": "blk.7.attn_k.bias", "offset": 5156397056, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 5156401152, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 5161119744, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.7.attn_q.bias", "offset": 5198868480, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 5198901248, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.7.attn_v.bias", "offset": 5236649984, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 5236654080, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 5243535360, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 5243568128, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 5500928000, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 5637177344, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 5637210112, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 5637242880, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.8.ffn_gate.weight", "offset": 5894602752, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 6030852096, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 6167101440, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 6167134208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 6167138304, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 6171856896, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.8.attn_q.bias", "offset": 6209605632, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 6209638400, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.8.attn_v.bias", "offset": 6247387136, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 6247391232, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 6254272512, "shape": [ 8192, 29568 ], "size": 6881280, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 6390521856, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.9.attn_k.bias", "offset": 6526771200, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 6526775296, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 6531493888, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.9.attn_q.bias", "offset": 6569242624, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 6569275392, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.9.attn_v.bias", "offset": 6607024128, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 6607028224, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 6613909504, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 6613942272, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.10.ffn_gate.weight", "offset": 6871302144, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 7007551488, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 7143800832, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 7143833600, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 7143837696, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 7148556288, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.10.attn_q.bias", "offset": 7186305024, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 7186337792, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.10.attn_v.bias", "offset": 7224086528, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 7224090624, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 7229857792, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 7229890560, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.11.ffn_gate.weight", "offset": 7396417536, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 7532666880, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 7668916224, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 7668948992, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 7668953088, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 7673671680, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.11.attn_q.bias", "offset": 7711420416, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 7711453184, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.11.attn_v.bias", "offset": 7749201920, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 7749206016, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 7754973184, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 7755005952, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.9.ffn_norm.weight", "offset": 7921532928, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 7921565696, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 7921598464, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.12.ffn_gate.weight", "offset": 8178958336, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 8315207680, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 8451457024, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 8451489792, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 8451493888, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 8456212480, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.12.attn_q.bias", "offset": 8493961216, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 8493993984, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.12.attn_v.bias", "offset": 8531742720, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 8531746816, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 8538628096, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 8538660864, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.13.ffn_gate.weight", "offset": 8705187840, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 8841437184, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 8977686528, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 8977719296, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 8977723392, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 8982441984, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.13.attn_q.bias", "offset": 9020190720, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 9020223488, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.13.attn_v.bias", "offset": 9057972224, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 9057976320, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_k.bias", "offset": 9063743488, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 9063747584, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 9068466176, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.14.attn_q.bias", "offset": 9106214912, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 9106247680, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.14.attn_v.bias", "offset": 9143996416, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 9144000512, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 9149767680, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 9149800448, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.14.ffn_gate.weight", "offset": 9316327424, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 9452576768, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 9588826112, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 9588858880, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 9588891648, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.15.ffn_gate.weight", "offset": 9846251520, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 9982500864, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 10118750208, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 10118782976, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 10118787072, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 10123505664, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.15.attn_q.bias", "offset": 10161254400, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 10161287168, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.15.attn_v.bias", "offset": 10199035904, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 10199040000, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 10205921280, "shape": [ 8192, 29568 ], "size": 6881280, "type": 12 }, { "name": "blk.16.attn_k.bias", "offset": 10342170624, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 10342174720, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 10346893312, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.16.attn_q.bias", "offset": 10384642048, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 10384674816, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.16.attn_v.bias", "offset": 10422423552, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 10422427648, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 10428194816, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 10428227584, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.16.ffn_up.weight", "offset": 10594754560, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 10731003904, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 10731036672, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 10731069440, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.17.ffn_gate.weight", "offset": 10897596416, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 11033845760, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 11170095104, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 11170127872, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 11170131968, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 11174850560, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.17.attn_q.bias", "offset": 11212599296, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 11212632064, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.17.attn_v.bias", "offset": 11250380800, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 11250384896, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 11256152064, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 11392401408, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.18.attn_k.bias", "offset": 11528650752, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 11528654848, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 11533373440, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.18.attn_q.bias", "offset": 11571122176, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 11571154944, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.18.attn_v.bias", "offset": 11608903680, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 11608907776, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 11615789056, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 11615821824, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 11873181696, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 11873214464, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 11873247232, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.19.ffn_gate.weight", "offset": 12039774208, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 12176023552, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.19.ffn_norm.weight", "offset": 12312272896, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 12312305664, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 12312309760, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 12317028352, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.19.attn_q.bias", "offset": 12354777088, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 12354809856, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.19.attn_v.bias", "offset": 12392558592, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 12392562688, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 12398329856, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 12398362624, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.20.ffn_gate.weight", "offset": 12564889600, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 12701138944, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 12837388288, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 12837421056, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 12837425152, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 12842143744, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.20.attn_q.bias", "offset": 12879892480, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 12879925248, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.20.attn_v.bias", "offset": 12917673984, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 12917678080, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 12923445248, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 12923478016, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.21.ffn_gate.weight", "offset": 13180837888, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 13317087232, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 13453336576, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 13453369344, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 13453373440, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 13458092032, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.21.attn_q.bias", "offset": 13495840768, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 13495873536, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.21.attn_v.bias", "offset": 13533622272, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 13533626368, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 13540507648, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 13540540416, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.22.ffn_gate.weight", "offset": 13707067392, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 13843316736, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 13979566080, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 13979598848, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 13979602944, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 13984321536, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.22.attn_q.bias", "offset": 14022070272, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 14022103040, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.22.attn_v.bias", "offset": 14059851776, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 14059855872, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_k.bias", "offset": 14065623040, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 14065627136, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 14070345728, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.23.attn_q.bias", "offset": 14108094464, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 14108127232, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.23.attn_v.bias", "offset": 14145875968, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 14145880064, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 14151647232, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 14151680000, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.23.ffn_gate.weight", "offset": 14318206976, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 14454456320, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 14590705664, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 14590738432, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 14590771200, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.24.ffn_gate.weight", "offset": 14848131072, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 14984380416, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 15120629760, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 15120662528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 15120666624, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 15125385216, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.24.attn_q.bias", "offset": 15163133952, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 15163166720, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.24.attn_v.bias", "offset": 15200915456, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 15200919552, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 15207800832, "shape": [ 8192, 29568 ], "size": 6881280, "type": 12 }, { "name": "blk.25.attn_k.bias", "offset": 15344050176, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 15344054272, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 15348772864, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.25.attn_q.bias", "offset": 15386521600, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 15386554368, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.25.attn_v.bias", "offset": 15424303104, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 15424307200, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 15430074368, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 15430107136, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.25.ffn_up.weight", "offset": 15596634112, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 15732883456, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 15732916224, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 15732948992, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.26.ffn_gate.weight", "offset": 15899475968, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 16035725312, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 16171974656, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 16172007424, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 16172011520, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 16176730112, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.26.attn_q.bias", "offset": 16214478848, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 16214511616, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.26.attn_v.bias", "offset": 16252260352, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 16252264448, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 16258031616, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 16394280960, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.27.attn_k.bias", "offset": 16530530304, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 16530534400, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 16535252992, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.27.attn_q.bias", "offset": 16573001728, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 16573034496, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.27.attn_v.bias", "offset": 16610783232, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 16610787328, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 16617668608, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 16617701376, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.27.ffn_norm.weight", "offset": 16875061248, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 16875094016, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 16875126784, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.28.ffn_gate.weight", "offset": 17041653760, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 17177903104, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.28.ffn_norm.weight", "offset": 17314152448, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 17314185216, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 17314189312, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 17318907904, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.28.attn_q.bias", "offset": 17356656640, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 17356689408, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.28.attn_v.bias", "offset": 17394438144, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 17394442240, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 17400209408, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 17400242176, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.29.ffn_gate.weight", "offset": 17566769152, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 17703018496, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.29.ffn_norm.weight", "offset": 17839267840, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 17839300608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 17839304704, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 17844023296, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.29.attn_q.bias", "offset": 17881772032, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 17881804800, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.29.attn_v.bias", "offset": 17919553536, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 17919557632, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 17925324800, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 17925357568, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.30.ffn_gate.weight", "offset": 18182717440, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 18318966784, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.30.ffn_norm.weight", "offset": 18455216128, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 18455248896, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 18455252992, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 18459971584, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.30.attn_q.bias", "offset": 18497720320, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 18497753088, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.30.attn_v.bias", "offset": 18535501824, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 18535505920, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.31.attn_norm.weight", "offset": 18542387200, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 18542419968, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.31.ffn_gate.weight", "offset": 18708946944, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 18845196288, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.31.ffn_norm.weight", "offset": 18981445632, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 18981478400, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 18981482496, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.31.attn_output.weight", "offset": 18986201088, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.31.attn_q.bias", "offset": 19023949824, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 19023982592, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.31.attn_v.bias", "offset": 19061731328, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 19061735424, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_k.bias", "offset": 19067502592, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 19067506688, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.32.attn_output.weight", "offset": 19072225280, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.32.attn_q.bias", "offset": 19109974016, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 19110006784, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.32.attn_v.bias", "offset": 19147755520, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 19147759616, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 19153526784, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 19153559552, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.32.ffn_gate.weight", "offset": 19320086528, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 19456335872, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.32.ffn_norm.weight", "offset": 19592585216, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 19592617984, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 19592650752, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.33.ffn_gate.weight", "offset": 19850010624, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 19986259968, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.33.ffn_norm.weight", "offset": 20122509312, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 20122542080, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 20122546176, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.33.attn_output.weight", "offset": 20127264768, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.33.attn_q.bias", "offset": 20165013504, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 20165046272, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.33.attn_v.bias", "offset": 20202795008, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 20202799104, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.34.ffn_gate.weight", "offset": 20209680384, "shape": [ 8192, 29568 ], "size": 6881280, "type": 12 }, { "name": "blk.34.attn_k.bias", "offset": 20345929728, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 20345933824, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.34.attn_output.weight", "offset": 20350652416, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.34.attn_q.bias", "offset": 20388401152, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 20388433920, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.34.attn_v.bias", "offset": 20426182656, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 20426186752, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 20431953920, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000\f\u0000\u0000\u0000\u0000Ў\\\u0006\u0000\u0000\u0000\u0012\u0000\u0000\u0000", "offset": 20431986688, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.34.ffn_up.weight", "offset": 20598513664, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.34.ffn_norm.weight", "offset": 20734763008, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 20734795776, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 20734828544, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.35.ffn_gate.weight", "offset": 20901355520, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 21037604864, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.35.ffn_norm.weight", "offset": 21173854208, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 21173886976, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 21173891072, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 21178609664, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.35.attn_q.bias", "offset": 21216358400, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 21216391168, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.35.attn_v.bias", "offset": 21254139904, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 21254144000, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 21259911168, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 21396160512, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.36.attn_k.bias", "offset": 21532409856, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 21532413952, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.36.attn_output.weight", "offset": 21537132544, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.36.attn_q.bias", "offset": 21574881280, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 21574914048, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.36.attn_v.bias", "offset": 21612662784, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 21612666880, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.36.attn_norm.weight", "offset": 21619548160, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 21619580928, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.36.ffn_norm.weight", "offset": 21876940800, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 21876973568, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 21877006336, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.37.ffn_gate.weight", "offset": 22043533312, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 22179782656, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.37.ffn_norm.weight", "offset": 22316032000, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 22316064768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 22316068864, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.37.attn_output.weight", "offset": 22320787456, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.37.attn_q.bias", "offset": 22358536192, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 22358568960, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.37.attn_v.bias", "offset": 22396317696, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 22396321792, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 22402088960, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 22402121728, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.38.ffn_gate.weight", "offset": 22568648704, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 22704898048, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.38.ffn_norm.weight", "offset": 22841147392, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 22841180160, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 22841184256, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.38.attn_output.weight", "offset": 22845902848, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.38.attn_q.bias", "offset": 22883651584, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 22883684352, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.38.attn_v.bias", "offset": 22921433088, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 22921437184, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 22927204352, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 22927237120, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.39.ffn_gate.weight", "offset": 23184596992, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 23320846336, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.39.ffn_norm.weight", "offset": 23457095680, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 23457128448, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 23457132544, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.39.attn_output.weight", "offset": 23461851136, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.39.attn_q.bias", "offset": 23499599872, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 23499632640, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.39.attn_v.bias", "offset": 23537381376, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 23537385472, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.40.attn_norm.weight", "offset": 23544266752, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 23544299520, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.40.ffn_gate.weight", "offset": 23710826496, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.40.ffn_up.weight", "offset": 23847075840, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.40.ffn_norm.weight", "offset": 23983325184, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 23983357952, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 23983362048, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.40.attn_output.weight", "offset": 23988080640, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.40.attn_q.bias", "offset": 24025829376, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 24025862144, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.40.attn_v.bias", "offset": 24063610880, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 24063614976, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_k.bias", "offset": 24069382144, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 24069386240, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.41.attn_output.weight", "offset": 24074104832, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.41.attn_q.bias", "offset": 24111853568, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 24111886336, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.41.attn_v.bias", "offset": 24149635072, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 24149639168, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 24155406336, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 24155439104, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.41.ffn_gate.weight", "offset": 24321966080, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.41.ffn_up.weight", "offset": 24458215424, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.41.ffn_norm.weight", "offset": 24594464768, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 24594497536, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 24594530304, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.42.ffn_gate.weight", "offset": 24851890176, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.42.ffn_up.weight", "offset": 24988139520, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.42.ffn_norm.weight", "offset": 25124388864, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 25124421632, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 25124425728, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.42.attn_output.weight", "offset": 25129144320, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.42.attn_q.bias", "offset": 25166893056, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 25166925824, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.42.attn_v.bias", "offset": 25204674560, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 25204678656, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.43.ffn_gate.weight", "offset": 25211559936, "shape": [ 8192, 29568 ], "size": 6881280, "type": 12 }, { "name": "blk.43.attn_k.bias", "offset": 25347809280, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 25347813376, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.43.attn_output.weight", "offset": 25352531968, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.43.attn_q.bias", "offset": 25390280704, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 25390313472, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.43.attn_v.bias", "offset": 25428062208, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 25428066304, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 25433833472, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 25433866240, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.43.ffn_up.weight", "offset": 25600393216, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.43.ffn_norm.weight", "offset": 25736642560, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 25736675328, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 25736708096, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.44.ffn_gate.weight", "offset": 25903235072, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.44.ffn_up.weight", "offset": 26039484416, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.44.ffn_norm.weight", "offset": 26175733760, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 26175766528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 26175770624, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.44.attn_output.weight", "offset": 26180489216, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.44.attn_q.bias", "offset": 26218237952, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 26218270720, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.44.attn_v.bias", "offset": 26256019456, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 26256023552, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.ffn_gate.weight", "offset": 26261790720, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.45.ffn_up.weight", "offset": 26398040064, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.45.attn_k.bias", "offset": 26534289408, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 26534293504, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.45.attn_output.weight", "offset": 26539012096, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.45.attn_q.bias", "offset": 26576760832, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 26576793600, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.45.attn_v.bias", "offset": 26614542336, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 26614546432, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.45.attn_norm.weight", "offset": 26621427712, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 26621460480, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.45.ffn_norm.weight", "offset": 26878820352, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 26878853120, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 26878885888, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.46.ffn_gate.weight", "offset": 27045412864, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.46.ffn_up.weight", "offset": 27181662208, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.46.ffn_norm.weight", "offset": 27317911552, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 27317944320, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 27317948416, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.46.attn_output.weight", "offset": 27322667008, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "\u0000\u0000\u0000blk.58.attn_out", "offset": 27360415744, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 27360448512, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.46.attn_v.bias", "offset": 27398197248, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 27398201344, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 27403968512, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 27404001280, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.47.ffn_gate.weight", "offset": 27570528256, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.47.ffn_up.weight", "offset": 27706777600, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.47.ffn_norm.weight", "offset": 27843026944, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 27843059712, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 27843063808, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.47.attn_output.weight", "offset": 27847782400, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.47.attn_q.bias", "offset": 27885531136, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 27885563904, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.47.attn_v.bias", "offset": 27923312640, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 27923316736, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.48.attn_norm.weight", "offset": 27929083904, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 27929116672, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.48.ffn_gate.weight", "offset": 28186476544, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.48.ffn_up.weight", "offset": 28322725888, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.48.ffn_norm.weight", "offset": 28458975232, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 28459008000, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 28459012096, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.48.attn_output.weight", "offset": 28463730688, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.48.attn_q.bias", "offset": 28501479424, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 28501512192, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.48.attn_v.bias", "offset": 28539260928, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 28539265024, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.49.attn_norm.weight", "offset": 28546146304, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 28546179072, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.49.ffn_gate.weight", "offset": 28712706048, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.49.ffn_up.weight", "offset": 28848955392, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.49.ffn_norm.weight", "offset": 28985204736, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 28985237504, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 28985241600, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.49.attn_output.weight", "offset": 28989960192, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.49.attn_q.bias", "offset": 29027708928, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 29027741696, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.49.attn_v.bias", "offset": 29065490432, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 29065494528, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_k.bias", "offset": 29071261696, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 29071265792, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.50.attn_output.weight", "offset": 29075984384, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.50.attn_q.bias", "offset": 29113733120, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 29113765888, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.50.attn_v.bias", "offset": 29151514624, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 29151518720, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_norm.weight", "offset": 29157285888, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 29157318656, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.50.ffn_gate.weight", "offset": 29323845632, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.50.ffn_up.weight", "offset": 29460094976, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.50.ffn_norm.weight", "offset": 29596344320, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 29596377088, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 29596409856, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.51.ffn_gate.weight", "offset": 29853769728, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.51.ffn_up.weight", "offset": 29990019072, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.51.ffn_norm.weight", "offset": 30126268416, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 30126301184, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 30126305280, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.51.attn_output.weight", "offset": 30131023872, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.51.attn_q.bias", "offset": 30168772608, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 30168805376, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.51.attn_v.bias", "offset": 30206554112, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 30206558208, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.52.ffn_gate.weight", "offset": 30213439488, "shape": [ 8192, 29568 ], "size": 6881280, "type": 12 }, { "name": "blk.52.attn_k.bias", "offset": 30349688832, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 30349692928, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.52.attn_output.weight", "offset": 30354411520, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.52.attn_q.bias", "offset": 30392160256, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 30392193024, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.52.attn_v.bias", "offset": 30429941760, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 30429945856, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.attn_norm.weight", "offset": 30435713024, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 30435745792, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.52.ffn_up.weight", "offset": 30602272768, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.52.ffn_norm.weight", "offset": 30738522112, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 30738554880, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 30738587648, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.53.ffn_gate.weight", "offset": 30905114624, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.53.ffn_up.weight", "offset": 31041363968, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.53.ffn_norm.weight", "offset": 31177613312, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 31177646080, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 31177650176, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.53.attn_output.weight", "offset": 31182368768, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.53.attn_q.bias", "offset": 31220117504, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 31220150272, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.53.attn_v.bias", "offset": 31257899008, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 31257903104, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.ffn_gate.weight", "offset": 31263670272, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.54.ffn_up.weight", "offset": 31399919616, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.54.attn_k.bias", "offset": 31536168960, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 31536173056, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.54.attn_output.weight", "offset": 31540891648, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.54.attn_q.bias", "offset": 31578640384, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 31578673152, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.54.attn_v.bias", "offset": 31616421888, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 31616425984, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.54.attn_norm.weight", "offset": 31623307264, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 31623340032, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.54.ffn_norm.weight", "offset": 31880699904, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 31880732672, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 31880765440, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.55.ffn_gate.weight", "offset": 32047292416, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.55.ffn_up.weight", "offset": 32183541760, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.55.ffn_norm.weight", "offset": 32319791104, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 32319823872, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 32319827968, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.55.attn_output.weight", "offset": 32324546560, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.55.attn_q.bias", "offset": 32362295296, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 32362328064, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.55.attn_v.bias", "offset": 32400076800, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 32400080896, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.56.attn_norm.weight", "offset": 32405848064, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 32405880832, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.56.ffn_gate.weight", "offset": 32572407808, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.56.ffn_up.weight", "offset": 32708657152, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.56.ffn_norm.weight", "offset": 32844906496, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 32844939264, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 32844943360, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.56.attn_output.weight", "offset": 32849661952, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.56.attn_q.bias", "offset": 32887410688, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 32887443456, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.56.attn_v.bias", "offset": 32925192192, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 32925196288, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.57.attn_norm.weight", "offset": 32930963456, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 32930996224, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.57.ffn_gate.weight", "offset": 33188356096, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.57.ffn_up.weight", "offset": 33324605440, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.57.ffn_norm.weight", "offset": 33460854784, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 33460887552, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 33460891648, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.57.attn_output.weight", "offset": 33465610240, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.57.attn_q.bias", "offset": 33503358976, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 33503391744, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.57.attn_v.bias", "offset": 33541140480, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 33541144576, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.58.attn_norm.weight", "offset": 33548025856, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 33548058624, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.58.ffn_gate.weight", "offset": 33714585600, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.58.ffn_up.weight", "offset": 33850834944, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.58.ffn_norm.weight", "offset": 33987084288, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 33987117056, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 33987121152, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.58.attn_output.weight", "offset": 33991839744, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.58.attn_q.bias", "offset": 34029588480, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 34029621248, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.58.attn_v.bias", "offset": 34067369984, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 34067374080, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_k.bias", "offset": 34073141248, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 34073145344, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.59.attn_output.weight", "offset": 34077863936, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.59.attn_q.bias", "offset": 34115612672, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 34115645440, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.59.attn_v.bias", "offset": 34153394176, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 34153398272, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_norm.weight", "offset": 34159165440, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 34159198208, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.59.ffn_gate.weight", "offset": 34325725184, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.59.ffn_up.weight", "offset": 34461974528, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.59.ffn_norm.weight", "offset": 34598223872, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 34598256640, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 34598289408, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.60.ffn_gate.weight", "offset": 34855649280, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.60.ffn_up.weight", "offset": 34991898624, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.60.ffn_norm.weight", "offset": 35128147968, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 35128180736, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 35128184832, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.60.attn_output.weight", "offset": 35132903424, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.60.attn_q.bias", "offset": 35170652160, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 35170684928, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.60.attn_v.bias", "offset": 35208433664, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 35208437760, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.61.ffn_gate.weight", "offset": 35215319040, "shape": [ 8192, 29568 ], "size": 6881280, "type": 12 }, { "name": "blk.61.attn_k.bias", "offset": 35351568384, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 35351572480, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.61.attn_output.weight", "offset": 35356291072, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.61.attn_q.bias", "offset": 35394039808, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 35394072576, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.61.attn_v.bias", "offset": 35431821312, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 35431825408, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.attn_norm.weight", "offset": 35437592576, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 35437625344, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.61.ffn_up.weight", "offset": 35604152320, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.61.ffn_norm.weight", "offset": 35740401664, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 35740434432, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 35740467200, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.62.ffn_gate.weight", "offset": 35906994176, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.62.ffn_up.weight", "offset": 36043243520, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.62.ffn_norm.weight", "offset": 36179492864, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 36179525632, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 36179529728, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.62.attn_output.weight", "offset": 36184248320, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.62.attn_q.bias", "offset": 36221997056, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 36222029824, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.62.attn_v.bias", "offset": 36259778560, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 36259782656, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.ffn_gate.weight", "offset": 36265549824, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.63.ffn_up.weight", "offset": 36401799168, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.63.attn_k.bias", "offset": 36538048512, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 36538052608, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.63.attn_output.weight", "offset": 36542771200, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.63.attn_q.bias", "offset": 36580519936, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 36580552704, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.63.attn_v.bias", "offset": 36618301440, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 36618305536, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.63.attn_norm.weight", "offset": 36625186816, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 36625219584, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.63.ffn_norm.weight", "offset": 36882579456, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 36882612224, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 36882644992, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.64.ffn_gate.weight", "offset": 37049171968, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.64.ffn_up.weight", "offset": 37185421312, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.64.ffn_norm.weight", "offset": 37321670656, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 37321703424, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 37321707520, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.64.attn_output.weight", "offset": 37326426112, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.64.attn_q.bias", "offset": 37364174848, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 37364207616, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.64.attn_v.bias", "offset": 37401956352, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 37401960448, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.65.attn_norm.weight", "offset": 37407727616, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 37407760384, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.65.ffn_gate.weight", "offset": 37574287360, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.65.ffn_up.weight", "offset": 37710536704, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.65.ffn_norm.weight", "offset": 37846786048, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 37846818816, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 37846822912, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.65.attn_output.weight", "offset": 37851541504, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.65.attn_q.bias", "offset": 37889290240, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 37889323008, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.65.attn_v.bias", "offset": 37927071744, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 37927075840, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.66.attn_norm.weight", "offset": 37932843008, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 37932875776, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.66.ffn_gate.weight", "offset": 38190235648, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.66.ffn_up.weight", "offset": 38326484992, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.66.ffn_norm.weight", "offset": 38462734336, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 38462767104, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 38462771200, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.66.attn_output.weight", "offset": 38467489792, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.66.attn_q.bias", "offset": 38505238528, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 38505271296, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.66.attn_v.bias", "offset": 38543020032, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 38543024128, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.67.attn_norm.weight", "offset": 38549905408, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 38549938176, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.67.ffn_gate.weight", "offset": 38716465152, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.67.ffn_up.weight", "offset": 38852714496, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.67.ffn_norm.weight", "offset": 38988963840, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 38988996608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 38989000704, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.67.attn_output.weight", "offset": 38993719296, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.67.attn_q.bias", "offset": 39031468032, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 39031500800, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.67.attn_v.bias", "offset": 39069249536, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 39069253632, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_k.bias", "offset": 39075020800, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 39075024896, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.68.attn_output.weight", "offset": 39079743488, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.68.attn_q.bias", "offset": 39117492224, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 39117524992, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.68.attn_v.bias", "offset": 39155273728, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 39155277824, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_norm.weight", "offset": 39161044992, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 39161077760, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.68.ffn_gate.weight", "offset": 39327604736, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.68.ffn_up.weight", "offset": 39463854080, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.68.ffn_norm.weight", "offset": 39600103424, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 39600136192, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 39600168960, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.69.ffn_gate.weight", "offset": 39857528832, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.69.ffn_up.weight", "offset": 39993778176, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.69.ffn_norm.weight", "offset": 40130027520, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 40130060288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 40130064384, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.69.attn_output.weight", "offset": 40134782976, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.69.attn_q.bias", "offset": 40172531712, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 40172564480, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.69.attn_v.bias", "offset": 40210313216, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 40210317312, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.70.ffn_gate.weight", "offset": 40217198592, "shape": [ 8192, 29568 ], "size": 6881280, "type": 12 }, { "name": "blk.70.attn_k.bias", "offset": 40353447936, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 40353452032, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.70.attn_output.weight", "offset": 40358170624, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.70.attn_q.bias", "offset": 40395919360, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 40395952128, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.70.attn_v.bias", "offset": 40433700864, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 40433704960, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.70.attn_norm.weight", "offset": 40440586240, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 40440619008, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.70.ffn_up.weight", "offset": 40697978880, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.70.ffn_norm.weight", "offset": 40834228224, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 40834260992, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 40834293760, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.71.ffn_gate.weight", "offset": 41091653632, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.71.ffn_up.weight", "offset": 41227902976, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.71.ffn_norm.weight", "offset": 41364152320, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 41364185088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 41364189184, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.71.attn_output.weight", "offset": 41368907776, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.71.attn_q.bias", "offset": 41406656512, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 41406689280, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.71.attn_v.bias", "offset": 41444438016, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 41444442112, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.72.ffn_gate.weight", "offset": 41451323392, "shape": [ 8192, 29568 ], "size": 6881280, "type": 12 }, { "name": "blk.72.ffn_up.weight", "offset": 41587572736, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.72.attn_k.bias", "offset": 41723822080, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 41723826176, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.72.attn_output.weight", "offset": 41728544768, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.72.attn_q.bias", "offset": 41766293504, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 41766326272, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.72.attn_v.bias", "offset": 41804075008, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 41804079104, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.72.attn_norm.weight", "offset": 41810960384, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 41810993152, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.72.ffn_norm.weight", "offset": 42068353024, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 42068385792, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 42068418560, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.73.ffn_gate.weight", "offset": 42325778432, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.73.ffn_up.weight", "offset": 42462027776, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.73.ffn_norm.weight", "offset": 42598277120, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 42598309888, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 42598313984, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.73.attn_output.weight", "offset": 42603032576, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.73.attn_q.bias", "offset": 42640781312, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 42640814080, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.73.attn_v.bias", "offset": 42678562816, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 42678566912, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.74.attn_norm.weight", "offset": 42685448192, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 42685480960, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.74.ffn_gate.weight", "offset": 42942840832, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.74.ffn_up.weight", "offset": 43079090176, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.74.ffn_norm.weight", "offset": 43215339520, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 43215372288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 43215376384, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.74.attn_output.weight", "offset": 43220094976, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.74.attn_q.bias", "offset": 43257843712, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 43257876480, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.74.attn_v.bias", "offset": 43295625216, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 43295629312, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.75.attn_norm.weight", "offset": 43302510592, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 43302543360, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.75.ffn_gate.weight", "offset": 43559903232, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.75.ffn_up.weight", "offset": 43696152576, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.75.ffn_norm.weight", "offset": 43832401920, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 43832434688, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 43832438784, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.75.attn_output.weight", "offset": 43837157376, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.75.attn_q.bias", "offset": 43874906112, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 43874938880, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.75.attn_v.bias", "offset": 43912687616, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 43912691712, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.76.attn_norm.weight", "offset": 43919572992, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 43919605760, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.76.ffn_gate.weight", "offset": 44176965632, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.76.ffn_up.weight", "offset": 44313214976, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.76.ffn_norm.weight", "offset": 44449464320, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 44449497088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 44449501184, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.76.attn_output.weight", "offset": 44454219776, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.76.attn_q.bias", "offset": 44491968512, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 44492001280, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.76.attn_v.bias", "offset": 44529750016, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 44529754112, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.77.attn_k.bias", "offset": 44536635392, "shape": [ 1024 ], "size": 6881280, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 44536639488, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.77.attn_output.weight", "offset": 44541358080, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.77.attn_q.bias", "offset": 44579106816, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 44579139584, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.77.attn_v.bias", "offset": 44616888320, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 44616892416, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.77.attn_norm.weight", "offset": 44623773696, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 44623806464, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.77.ffn_gate.weight", "offset": 44881166336, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.77.ffn_up.weight", "offset": 45017415680, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.77.ffn_norm.weight", "offset": 45153665024, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 45153697792, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 45153730560, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.78.ffn_gate.weight", "offset": 45411090432, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.78.ffn_up.weight", "offset": 45547339776, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.78.ffn_norm.weight", "offset": 45683589120, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 45683621888, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 45683625984, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.78.attn_output.weight", "offset": 45688344576, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.78.attn_q.bias", "offset": 45726093312, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 45726126080, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.78.attn_v.bias", "offset": 45763874816, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 45763878912, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.79.ffn_gate.weight", "offset": 45770760192, "shape": [ 8192, 29568 ], "size": 6881280, "type": 12 }, { "name": "blk.79.attn_k.bias", "offset": 45907009536, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 45907013632, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.79.attn_output.weight", "offset": 45911732224, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.79.attn_q.bias", "offset": 45949480960, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 45949513728, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.79.attn_v.bias", "offset": 45987262464, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 45987266560, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "output.weight", "offset": 45994147840, "shape": [ 8192, 152064 ], "size": 6881280, "type": 14 }, { "name": "blk.79.attn_norm.weight", "offset": 47016017920, "shape": [ 8192 ], "size": 1021870080, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 47016050688, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.79.ffn_up.weight", "offset": 47273410560, "shape": [ 8192, 29568 ], "size": 257359872, "type": 12 }, { "name": "blk.79.ffn_norm.weight", "offset": 47409659904, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "output_norm.weight", "offset": 47409692672, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }