qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 3, "general.name": "Qwen2-72B", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.00001, "qwen2.block_count": 80, "qwen2.context_length": 131072, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 778567680, "shape": [ 8192, 29568 ], "size": 778567680, "type": 3 }, { "name": "blk.0.ffn_up.weight", "offset": 929955840, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.0.attn_k.bias", "offset": 1081344000, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 1081348096, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.0.attn_output.weight", "offset": 1086590976, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.0.attn_q.bias", "offset": 1128534016, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 1128566784, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.0.attn_v.bias", "offset": 1170509824, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 1170513920, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.0.attn_norm.weight", "offset": 1175756800, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 1175789568, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.0.ffn_norm.weight", "offset": 1327177728, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1327210496, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1327243264, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.1.ffn_gate.weight", "offset": 1478631424, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.1.ffn_up.weight", "offset": 1630019584, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.1.ffn_norm.weight", "offset": 1781407744, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 1781440512, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1781444608, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.1.attn_output.weight", "offset": 1786687488, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.1.attn_q.bias", "offset": 1828630528, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 1828663296, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.1.attn_v.bias", "offset": 1870606336, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 1870610432, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.2.attn_norm.weight", "offset": 1875853312, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1875886080, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.2.ffn_gate.weight", "offset": 2027274240, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.2.ffn_up.weight", "offset": 2178662400, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.2.ffn_norm.weight", "offset": 2330050560, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 2330083328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 2330087424, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.2.attn_output.weight", "offset": 2335330304, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.2.attn_q.bias", "offset": 2377273344, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 2377306112, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.2.attn_v.bias", "offset": 2419249152, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 2419253248, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.3.attn_norm.weight", "offset": 2424496128, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 2424528896, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.3.ffn_gate.weight", "offset": 2575917056, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.3.ffn_up.weight", "offset": 2727305216, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.3.ffn_norm.weight", "offset": 2878693376, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 2878726144, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 2878730240, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.3.attn_output.weight", "offset": 2883973120, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.3.attn_q.bias", "offset": 2925916160, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 2925948928, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.3.attn_v.bias", "offset": 2967891968, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 2967896064, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.4.attn_norm.weight", "offset": 2973138944, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 2973171712, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.4.ffn_gate.weight", "offset": 3124559872, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.4.ffn_up.weight", "offset": 3275948032, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.4.ffn_norm.weight", "offset": 3427336192, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 3427368960, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 3427373056, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.4.attn_output.weight", "offset": 3432615936, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.4.attn_q.bias", "offset": 3474558976, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 3474591744, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.4.attn_v.bias", "offset": 3516534784, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 3516538880, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.5.attn_k.bias", "offset": 3521781760, "shape": [ 1024 ], "size": 5242880, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 3521785856, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.5.attn_output.weight", "offset": 3527028736, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.5.attn_q.bias", "offset": 3568971776, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 3569004544, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.5.attn_v.bias", "offset": 3610947584, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 3610951680, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.5.attn_norm.weight", "offset": 3616194560, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 3616227328, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.5.ffn_gate.weight", "offset": 3767615488, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.5.ffn_up.weight", "offset": 3919003648, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.5.ffn_norm.weight", "offset": 4070391808, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 4070424576, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 4070457344, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.6.ffn_gate.weight", "offset": 4221845504, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.6.ffn_up.weight", "offset": 4373233664, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.6.ffn_norm.weight", "offset": 4524621824, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 4524654592, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 4524658688, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.6.attn_output.weight", "offset": 4529901568, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.6.attn_q.bias", "offset": 4571844608, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 4571877376, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.6.attn_v.bias", "offset": 4613820416, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 4613824512, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.7.ffn_gate.weight", "offset": 4619067392, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.7.attn_k.bias", "offset": 4770455552, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 4770459648, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.7.attn_output.weight", "offset": 4775702528, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.7.attn_q.bias", "offset": 4817645568, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 4817678336, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.7.attn_v.bias", "offset": 4859621376, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 4859625472, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.7.attn_norm.weight", "offset": 4864868352, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 4864901120, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.7.ffn_up.weight", "offset": 5016289280, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.7.ffn_norm.weight", "offset": 5167677440, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 5167710208, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 5167742976, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.8.ffn_gate.weight", "offset": 5319131136, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.8.ffn_up.weight", "offset": 5470519296, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.8.ffn_norm.weight", "offset": 5621907456, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 5621940224, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 5621944320, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.8.attn_output.weight", "offset": 5627187200, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.8.attn_q.bias", "offset": 5669130240, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 5669163008, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.8.attn_v.bias", "offset": 5711106048, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 5711110144, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.9.ffn_gate.weight", "offset": 5716353024, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.9.ffn_up.weight", "offset": 5867741184, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.9.attn_k.bias", "offset": 6019129344, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 6019133440, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.9.attn_output.weight", "offset": 6024376320, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.9.attn_q.bias", "offset": 6066319360, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 6066352128, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.9.attn_v.bias", "offset": 6108295168, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 6108299264, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.10.attn_norm.weight", "offset": 6113542144, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 6113574912, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.10.ffn_gate.weight", "offset": 6264963072, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.10.ffn_up.weight", "offset": 6416351232, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.10.ffn_norm.weight", "offset": 6567739392, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 6567772160, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 6567776256, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.10.attn_output.weight", "offset": 6573019136, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.10.attn_q.bias", "offset": 6614962176, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 6614994944, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.10.attn_v.bias", "offset": 6656937984, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 6656942080, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.11.attn_norm.weight", "offset": 6662184960, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 6662217728, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.11.ffn_gate.weight", "offset": 6813605888, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.11.ffn_up.weight", "offset": 6964994048, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.11.ffn_norm.weight", "offset": 7116382208, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 7116414976, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 7116419072, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.11.attn_output.weight", "offset": 7121661952, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.11.attn_q.bias", "offset": 7163604992, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 7163637760, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.11.attn_v.bias", "offset": 7205580800, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 7205584896, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.9.attn_norm.weight", "offset": 7210827776, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 7210860544, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.9.ffn_norm.weight", "offset": 7362248704, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 7362281472, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 7362314240, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.12.ffn_gate.weight", "offset": 7513702400, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.12.ffn_up.weight", "offset": 7665090560, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.12.ffn_norm.weight", "offset": 7816478720, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 7816511488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 7816515584, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.12.attn_output.weight", "offset": 7821758464, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.12.attn_q.bias", "offset": 7863701504, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 7863734272, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.12.attn_v.bias", "offset": 7905677312, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 7905681408, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.13.attn_norm.weight", "offset": 7910924288, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 7910957056, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.13.ffn_gate.weight", "offset": 8062345216, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.13.ffn_up.weight", "offset": 8213733376, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.13.ffn_norm.weight", "offset": 8365121536, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 8365154304, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 8365158400, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.13.attn_output.weight", "offset": 8370401280, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.13.attn_q.bias", "offset": 8412344320, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 8412377088, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.13.attn_v.bias", "offset": 8454320128, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 8454324224, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.14.attn_k.bias", "offset": 8459567104, "shape": [ 1024 ], "size": 5242880, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 8459571200, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.14.attn_output.weight", "offset": 8464814080, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.14.attn_q.bias", "offset": 8506757120, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 8506789888, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.14.attn_v.bias", "offset": 8548732928, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 8548737024, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.14.attn_norm.weight", "offset": 8553979904, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 8554012672, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.14.ffn_gate.weight", "offset": 8705400832, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.14.ffn_up.weight", "offset": 8856788992, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.14.ffn_norm.weight", "offset": 9008177152, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 9008209920, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 9008242688, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.15.ffn_gate.weight", "offset": 9159630848, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.15.ffn_up.weight", "offset": 9311019008, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.15.ffn_norm.weight", "offset": 9462407168, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 9462439936, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 9462444032, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.15.attn_output.weight", "offset": 9467686912, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.15.attn_q.bias", "offset": 9509629952, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 9509662720, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.15.attn_v.bias", "offset": 9551605760, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 9551609856, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.16.ffn_gate.weight", "offset": 9556852736, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.16.attn_k.bias", "offset": 9708240896, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 9708244992, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.16.attn_output.weight", "offset": 9713487872, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.16.attn_q.bias", "offset": 9755430912, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 9755463680, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.16.attn_v.bias", "offset": 9797406720, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 9797410816, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.16.attn_norm.weight", "offset": 9802653696, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 9802686464, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.16.ffn_up.weight", "offset": 9954074624, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.16.ffn_norm.weight", "offset": 10105462784, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 10105495552, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 10105528320, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.17.ffn_gate.weight", "offset": 10256916480, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.17.ffn_up.weight", "offset": 10408304640, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.17.ffn_norm.weight", "offset": 10559692800, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 10559725568, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 10559729664, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.17.attn_output.weight", "offset": 10564972544, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.17.attn_q.bias", "offset": 10606915584, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 10606948352, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.17.attn_v.bias", "offset": 10648891392, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 10648895488, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.18.ffn_gate.weight", "offset": 10654138368, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.18.ffn_up.weight", "offset": 10805526528, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.18.attn_k.bias", "offset": 10956914688, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 10956918784, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.18.attn_output.weight", "offset": 10962161664, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.18.attn_q.bias", "offset": 11004104704, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 11004137472, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.18.attn_v.bias", "offset": 11046080512, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 11046084608, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.18.attn_norm.weight", "offset": 11051327488, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 11051360256, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.18.ffn_norm.weight", "offset": 11202748416, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 11202781184, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 11202813952, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.19.ffn_gate.weight", "offset": 11354202112, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.19.ffn_up.weight", "offset": 11505590272, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.19.ffn_norm.weight", "offset": 11656978432, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 11657011200, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 11657015296, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.19.attn_output.weight", "offset": 11662258176, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.19.attn_q.bias", "offset": 11704201216, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 11704233984, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.19.attn_v.bias", "offset": 11746177024, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 11746181120, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.20.attn_norm.weight", "offset": 11751424000, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 11751456768, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.20.ffn_gate.weight", "offset": 11902844928, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.20.ffn_up.weight", "offset": 12054233088, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.20.ffn_norm.weight", "offset": 12205621248, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 12205654016, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 12205658112, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.20.attn_output.weight", "offset": 12210900992, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.20.attn_q.bias", "offset": 12252844032, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 12252876800, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.20.attn_v.bias", "offset": 12294819840, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 12294823936, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.21.attn_norm.weight", "offset": 12300066816, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 12300099584, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.21.ffn_gate.weight", "offset": 12451487744, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.21.ffn_up.weight", "offset": 12602875904, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.21.ffn_norm.weight", "offset": 12754264064, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 12754296832, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 12754300928, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.21.attn_output.weight", "offset": 12759543808, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.21.attn_q.bias", "offset": 12801486848, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 12801519616, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.21.attn_v.bias", "offset": 12843462656, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 12843466752, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.22.attn_norm.weight", "offset": 12848709632, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 12848742400, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.22.ffn_gate.weight", "offset": 13000130560, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.22.ffn_up.weight", "offset": 13151518720, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.22.ffn_norm.weight", "offset": 13302906880, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 13302939648, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 13302943744, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.22.attn_output.weight", "offset": 13308186624, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.22.attn_q.bias", "offset": 13350129664, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 13350162432, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.22.attn_v.bias", "offset": 13392105472, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 13392109568, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.23.attn_k.bias", "offset": 13397352448, "shape": [ 1024 ], "size": 5242880, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 13397356544, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.23.attn_output.weight", "offset": 13402599424, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.23.attn_q.bias", "offset": 13444542464, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 13444575232, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.23.attn_v.bias", "offset": 13486518272, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 13486522368, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.23.attn_norm.weight", "offset": 13491765248, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 13491798016, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.23.ffn_gate.weight", "offset": 13643186176, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.23.ffn_up.weight", "offset": 13794574336, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.23.ffn_norm.weight", "offset": 13945962496, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 13945995264, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 13946028032, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.24.ffn_gate.weight", "offset": 14097416192, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.24.ffn_up.weight", "offset": 14248804352, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.24.ffn_norm.weight", "offset": 14400192512, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 14400225280, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 14400229376, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.24.attn_output.weight", "offset": 14405472256, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.24.attn_q.bias", "offset": 14447415296, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 14447448064, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.24.attn_v.bias", "offset": 14489391104, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 14489395200, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.25.ffn_gate.weight", "offset": 14494638080, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.25.attn_k.bias", "offset": 14646026240, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 14646030336, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.25.attn_output.weight", "offset": 14651273216, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.25.attn_q.bias", "offset": 14693216256, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 14693249024, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.25.attn_v.bias", "offset": 14735192064, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 14735196160, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.25.attn_norm.weight", "offset": 14740439040, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 14740471808, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.25.ffn_up.weight", "offset": 14891859968, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.25.ffn_norm.weight", "offset": 15043248128, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 15043280896, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 15043313664, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.26.ffn_gate.weight", "offset": 15194701824, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.26.ffn_up.weight", "offset": 15346089984, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.26.ffn_norm.weight", "offset": 15497478144, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 15497510912, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 15497515008, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.26.attn_output.weight", "offset": 15502757888, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.26.attn_q.bias", "offset": 15544700928, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 15544733696, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.26.attn_v.bias", "offset": 15586676736, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 15586680832, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.27.ffn_gate.weight", "offset": 15591923712, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.27.ffn_up.weight", "offset": 15743311872, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.27.attn_k.bias", "offset": 15894700032, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 15894704128, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.27.attn_output.weight", "offset": 15899947008, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.27.attn_q.bias", "offset": 15941890048, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 15941922816, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.27.attn_v.bias", "offset": 15983865856, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 15983869952, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.27.attn_norm.weight", "offset": 15989112832, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 15989145600, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.27.ffn_norm.weight", "offset": 16140533760, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 16140566528, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 16140599296, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.28.ffn_gate.weight", "offset": 16291987456, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.28.ffn_up.weight", "offset": 16443375616, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.28.ffn_norm.weight", "offset": 16594763776, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 16594796544, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 16594800640, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.28.attn_output.weight", "offset": 16600043520, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.28.attn_q.bias", "offset": 16641986560, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 16642019328, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.28.attn_v.bias", "offset": 16683962368, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 16683966464, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.29.attn_norm.weight", "offset": 16689209344, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 16689242112, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.29.ffn_gate.weight", "offset": 16840630272, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.29.ffn_up.weight", "offset": 16992018432, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.29.ffn_norm.weight", "offset": 17143406592, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 17143439360, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 17143443456, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.29.attn_output.weight", "offset": 17148686336, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.29.attn_q.bias", "offset": 17190629376, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 17190662144, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.29.attn_v.bias", "offset": 17232605184, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 17232609280, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.30.attn_norm.weight", "offset": 17237852160, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 17237884928, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.30.ffn_gate.weight", "offset": 17389273088, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.30.ffn_up.weight", "offset": 17540661248, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.30.ffn_norm.weight", "offset": 17692049408, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 17692082176, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 17692086272, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.30.attn_output.weight", "offset": 17697329152, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.30.attn_q.bias", "offset": 17739272192, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 17739304960, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.30.attn_v.bias", "offset": 17781248000, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 17781252096, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.31.attn_norm.weight", "offset": 17786494976, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 17786527744, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.31.ffn_gate.weight", "offset": 17937915904, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.31.ffn_up.weight", "offset": 18089304064, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.31.ffn_norm.weight", "offset": 18240692224, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 18240724992, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 18240729088, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.31.attn_output.weight", "offset": 18245971968, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.31.attn_q.bias", "offset": 18287915008, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 18287947776, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.31.attn_v.bias", "offset": 18329890816, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 18329894912, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.32.attn_k.bias", "offset": 18335137792, "shape": [ 1024 ], "size": 5242880, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 18335141888, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.32.attn_output.weight", "offset": 18340384768, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.32.attn_q.bias", "offset": 18382327808, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 18382360576, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.32.attn_v.bias", "offset": 18424303616, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 18424307712, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.32.attn_norm.weight", "offset": 18429550592, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 18429583360, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.32.ffn_gate.weight", "offset": 18580971520, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.32.ffn_up.weight", "offset": 18732359680, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.32.ffn_norm.weight", "offset": 18883747840, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 18883780608, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 18883813376, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.33.ffn_gate.weight", "offset": 19035201536, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.33.ffn_up.weight", "offset": 19186589696, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.33.ffn_norm.weight", "offset": 19337977856, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 19338010624, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 19338014720, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.33.attn_output.weight", "offset": 19343257600, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.33.attn_q.bias", "offset": 19385200640, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 19385233408, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.33.attn_v.bias", "offset": 19427176448, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 19427180544, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.34.ffn_gate.weight", "offset": 19432423424, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.34.attn_k.bias", "offset": 19583811584, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 19583815680, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.34.attn_output.weight", "offset": 19589058560, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.34.attn_q.bias", "offset": 19631001600, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 19631034368, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.34.attn_v.bias", "offset": 19672977408, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 19672981504, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.34.attn_norm.weight", "offset": 19678224384, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 19678257152, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.34.ffn_up.weight", "offset": 19829645312, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.34.ffn_norm.weight", "offset": 19981033472, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 19981066240, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 19981099008, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.35.ffn_gate.weight", "offset": 20132487168, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.35.ffn_up.weight", "offset": 20283875328, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.35.ffn_norm.weight", "offset": 20435263488, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 20435296256, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 20435300352, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.35.attn_output.weight", "offset": 20440543232, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.35.attn_q.bias", "offset": 20482486272, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 20482519040, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.35.attn_v.bias", "offset": 20524462080, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 20524466176, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.36.ffn_gate.weight", "offset": 20529709056, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.36.ffn_up.weight", "offset": 20681097216, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.36.attn_k.bias", "offset": 20832485376, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 20832489472, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.36.attn_output.weight", "offset": 20837732352, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.36.attn_q.bias", "offset": 20879675392, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 20879708160, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.36.attn_v.bias", "offset": 20921651200, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 20921655296, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.36.attn_norm.weight", "offset": 20926898176, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 20926930944, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.36.ffn_norm.weight", "offset": 21078319104, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 21078351872, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 21078384640, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.37.ffn_gate.weight", "offset": 21229772800, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.37.ffn_up.weight", "offset": 21381160960, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.37.ffn_norm.weight", "offset": 21532549120, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 21532581888, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 21532585984, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.37.attn_output.weight", "offset": 21537828864, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.37.attn_q.bias", "offset": 21579771904, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 21579804672, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.37.attn_v.bias", "offset": 21621747712, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 21621751808, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.38.attn_norm.weight", "offset": 21626994688, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 21627027456, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.38.ffn_gate.weight", "offset": 21778415616, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.38.ffn_up.weight", "offset": 21929803776, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.38.ffn_norm.weight", "offset": 22081191936, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 22081224704, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 22081228800, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.38.attn_output.weight", "offset": 22086471680, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.38.attn_q.bias", "offset": 22128414720, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 22128447488, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.38.attn_v.bias", "offset": 22170390528, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 22170394624, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.39.attn_norm.weight", "offset": 22175637504, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 22175670272, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.39.ffn_gate.weight", "offset": 22327058432, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.39.ffn_up.weight", "offset": 22478446592, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.39.ffn_norm.weight", "offset": 22629834752, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 22629867520, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 22629871616, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.39.attn_output.weight", "offset": 22635114496, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.39.attn_q.bias", "offset": 22677057536, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 22677090304, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.39.attn_v.bias", "offset": 22719033344, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 22719037440, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.40.attn_norm.weight", "offset": 22724280320, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 22724313088, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.40.ffn_gate.weight", "offset": 22875701248, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.40.ffn_up.weight", "offset": 23027089408, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.40.ffn_norm.weight", "offset": 23178477568, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 23178510336, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 23178514432, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.40.attn_output.weight", "offset": 23183757312, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.40.attn_q.bias", "offset": 23225700352, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 23225733120, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.40.attn_v.bias", "offset": 23267676160, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 23267680256, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.41.attn_k.bias", "offset": 23272923136, "shape": [ 1024 ], "size": 5242880, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 23272927232, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.41.attn_output.weight", "offset": 23278170112, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.41.attn_q.bias", "offset": 23320113152, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 23320145920, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.41.attn_v.bias", "offset": 23362088960, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 23362093056, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.41.attn_norm.weight", "offset": 23367335936, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 23367368704, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.41.ffn_gate.weight", "offset": 23518756864, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.41.ffn_up.weight", "offset": 23670145024, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.41.ffn_norm.weight", "offset": 23821533184, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 23821565952, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 23821598720, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.42.ffn_gate.weight", "offset": 23972986880, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.42.ffn_up.weight", "offset": 24124375040, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.42.ffn_norm.weight", "offset": 24275763200, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 24275795968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 24275800064, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.42.attn_output.weight", "offset": 24281042944, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.42.attn_q.bias", "offset": 24322985984, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 24323018752, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.42.attn_v.bias", "offset": 24364961792, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 24364965888, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.43.ffn_gate.weight", "offset": 24370208768, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.43.attn_k.bias", "offset": 24521596928, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 24521601024, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.43.attn_output.weight", "offset": 24526843904, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.43.attn_q.bias", "offset": 24568786944, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 24568819712, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.43.attn_v.bias", "offset": 24610762752, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 24610766848, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.43.attn_norm.weight", "offset": 24616009728, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 24616042496, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.43.ffn_up.weight", "offset": 24767430656, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.43.ffn_norm.weight", "offset": 24918818816, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 24918851584, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 24918884352, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.44.ffn_gate.weight", "offset": 25070272512, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.44.ffn_up.weight", "offset": 25221660672, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.44.ffn_norm.weight", "offset": 25373048832, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 25373081600, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 25373085696, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.44.attn_output.weight", "offset": 25378328576, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.44.attn_q.bias", "offset": 25420271616, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 25420304384, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.44.attn_v.bias", "offset": 25462247424, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 25462251520, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.45.ffn_gate.weight", "offset": 25467494400, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.45.ffn_up.weight", "offset": 25618882560, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.45.attn_k.bias", "offset": 25770270720, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 25770274816, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.45.attn_output.weight", "offset": 25775517696, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.45.attn_q.bias", "offset": 25817460736, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 25817493504, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.45.attn_v.bias", "offset": 25859436544, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 25859440640, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.45.attn_norm.weight", "offset": 25864683520, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 25864716288, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.45.ffn_norm.weight", "offset": 26016104448, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 26016137216, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 26016169984, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.46.ffn_gate.weight", "offset": 26167558144, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.46.ffn_up.weight", "offset": 26318946304, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.46.ffn_norm.weight", "offset": 26470334464, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 26470367232, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 26470371328, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.46.attn_output.weight", "offset": 26475614208, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "lk.58.attn_output.", "offset": 26517557248, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 26517590016, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.46.attn_v.bias", "offset": 26559533056, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 26559537152, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.47.attn_norm.weight", "offset": 26564780032, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 26564812800, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.47.ffn_gate.weight", "offset": 26716200960, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.47.ffn_up.weight", "offset": 26867589120, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.47.ffn_norm.weight", "offset": 27018977280, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 27019010048, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 27019014144, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.47.attn_output.weight", "offset": 27024257024, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.47.attn_q.bias", "offset": 27066200064, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 27066232832, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.47.attn_v.bias", "offset": 27108175872, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 27108179968, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.48.attn_norm.weight", "offset": 27113422848, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 27113455616, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.48.ffn_gate.weight", "offset": 27264843776, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.48.ffn_up.weight", "offset": 27416231936, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.48.ffn_norm.weight", "offset": 27567620096, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 27567652864, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 27567656960, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.48.attn_output.weight", "offset": 27572899840, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.48.attn_q.bias", "offset": 27614842880, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 27614875648, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.48.attn_v.bias", "offset": 27656818688, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 27656822784, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.49.attn_norm.weight", "offset": 27662065664, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 27662098432, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.49.ffn_gate.weight", "offset": 27813486592, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.49.ffn_up.weight", "offset": 27964874752, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.49.ffn_norm.weight", "offset": 28116262912, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 28116295680, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 28116299776, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.49.attn_output.weight", "offset": 28121542656, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.49.attn_q.bias", "offset": 28163485696, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 28163518464, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.49.attn_v.bias", "offset": 28205461504, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 28205465600, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.50.attn_k.bias", "offset": 28210708480, "shape": [ 1024 ], "size": 5242880, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 28210712576, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.50.attn_output.weight", "offset": 28215955456, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.50.attn_q.bias", "offset": 28257898496, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 28257931264, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.50.attn_v.bias", "offset": 28299874304, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 28299878400, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.50.attn_norm.weight", "offset": 28305121280, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 28305154048, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.50.ffn_gate.weight", "offset": 28456542208, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.50.ffn_up.weight", "offset": 28607930368, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.50.ffn_norm.weight", "offset": 28759318528, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 28759351296, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 28759384064, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.51.ffn_gate.weight", "offset": 28910772224, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.51.ffn_up.weight", "offset": 29062160384, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.51.ffn_norm.weight", "offset": 29213548544, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 29213581312, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 29213585408, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.51.attn_output.weight", "offset": 29218828288, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.51.attn_q.bias", "offset": 29260771328, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 29260804096, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.51.attn_v.bias", "offset": 29302747136, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 29302751232, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.52.ffn_gate.weight", "offset": 29307994112, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.52.attn_k.bias", "offset": 29459382272, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 29459386368, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.52.attn_output.weight", "offset": 29464629248, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.52.attn_q.bias", "offset": 29506572288, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 29506605056, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.52.attn_v.bias", "offset": 29548548096, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 29548552192, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.52.attn_norm.weight", "offset": 29553795072, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 29553827840, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.52.ffn_up.weight", "offset": 29705216000, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.52.ffn_norm.weight", "offset": 29856604160, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 29856636928, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 29856669696, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.53.ffn_gate.weight", "offset": 30008057856, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.53.ffn_up.weight", "offset": 30159446016, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.53.ffn_norm.weight", "offset": 30310834176, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 30310866944, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 30310871040, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.53.attn_output.weight", "offset": 30316113920, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.53.attn_q.bias", "offset": 30358056960, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 30358089728, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.53.attn_v.bias", "offset": 30400032768, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 30400036864, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.54.ffn_gate.weight", "offset": 30405279744, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.54.ffn_up.weight", "offset": 30556667904, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.54.attn_k.bias", "offset": 30708056064, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 30708060160, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.54.attn_output.weight", "offset": 30713303040, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.54.attn_q.bias", "offset": 30755246080, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 30755278848, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.54.attn_v.bias", "offset": 30797221888, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 30797225984, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.54.attn_norm.weight", "offset": 30802468864, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 30802501632, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.54.ffn_norm.weight", "offset": 30953889792, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 30953922560, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 30953955328, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.55.ffn_gate.weight", "offset": 31105343488, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.55.ffn_up.weight", "offset": 31256731648, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.55.ffn_norm.weight", "offset": 31408119808, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 31408152576, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 31408156672, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.55.attn_output.weight", "offset": 31413399552, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.55.attn_q.bias", "offset": 31455342592, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 31455375360, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.55.attn_v.bias", "offset": 31497318400, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 31497322496, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.56.attn_norm.weight", "offset": 31502565376, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 31502598144, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.56.ffn_gate.weight", "offset": 31653986304, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.56.ffn_up.weight", "offset": 31805374464, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.56.ffn_norm.weight", "offset": 31956762624, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 31956795392, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 31956799488, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.56.attn_output.weight", "offset": 31962042368, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.56.attn_q.bias", "offset": 32003985408, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 32004018176, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.56.attn_v.bias", "offset": 32045961216, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 32045965312, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.57.attn_norm.weight", "offset": 32051208192, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 32051240960, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.57.ffn_gate.weight", "offset": 32202629120, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.57.ffn_up.weight", "offset": 32354017280, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.57.ffn_norm.weight", "offset": 32505405440, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 32505438208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 32505442304, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.57.attn_output.weight", "offset": 32510685184, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.57.attn_q.bias", "offset": 32552628224, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 32552660992, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.57.attn_v.bias", "offset": 32594604032, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 32594608128, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.58.attn_norm.weight", "offset": 32599851008, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 32599883776, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.58.ffn_gate.weight", "offset": 32751271936, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.58.ffn_up.weight", "offset": 32902660096, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.58.ffn_norm.weight", "offset": 33054048256, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 33054081024, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 33054085120, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.58.attn_output.weight", "offset": 33059328000, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.58.attn_q.bias", "offset": 33101271040, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 33101303808, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.58.attn_v.bias", "offset": 33143246848, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 33143250944, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.59.attn_k.bias", "offset": 33148493824, "shape": [ 1024 ], "size": 5242880, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 33148497920, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.59.attn_output.weight", "offset": 33153740800, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.59.attn_q.bias", "offset": 33195683840, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 33195716608, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.59.attn_v.bias", "offset": 33237659648, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 33237663744, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.59.attn_norm.weight", "offset": 33242906624, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 33242939392, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.59.ffn_gate.weight", "offset": 33394327552, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.59.ffn_up.weight", "offset": 33545715712, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.59.ffn_norm.weight", "offset": 33697103872, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 33697136640, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 33697169408, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.60.ffn_gate.weight", "offset": 33848557568, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.60.ffn_up.weight", "offset": 33999945728, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.60.ffn_norm.weight", "offset": 34151333888, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 34151366656, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 34151370752, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.60.attn_output.weight", "offset": 34156613632, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.60.attn_q.bias", "offset": 34198556672, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 34198589440, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.60.attn_v.bias", "offset": 34240532480, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 34240536576, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.61.ffn_gate.weight", "offset": 34245779456, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.61.attn_k.bias", "offset": 34397167616, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 34397171712, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.61.attn_output.weight", "offset": 34402414592, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.61.attn_q.bias", "offset": 34444357632, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 34444390400, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.61.attn_v.bias", "offset": 34486333440, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 34486337536, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.61.attn_norm.weight", "offset": 34491580416, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 34491613184, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.61.ffn_up.weight", "offset": 34643001344, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.61.ffn_norm.weight", "offset": 34794389504, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 34794422272, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 34794455040, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.62.ffn_gate.weight", "offset": 34945843200, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.62.ffn_up.weight", "offset": 35097231360, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.62.ffn_norm.weight", "offset": 35248619520, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 35248652288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 35248656384, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.62.attn_output.weight", "offset": 35253899264, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.62.attn_q.bias", "offset": 35295842304, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 35295875072, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.62.attn_v.bias", "offset": 35337818112, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 35337822208, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.63.ffn_gate.weight", "offset": 35343065088, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.63.ffn_up.weight", "offset": 35494453248, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.63.attn_k.bias", "offset": 35645841408, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 35645845504, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.63.attn_output.weight", "offset": 35651088384, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.63.attn_q.bias", "offset": 35693031424, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 35693064192, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.63.attn_v.bias", "offset": 35735007232, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 35735011328, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.63.attn_norm.weight", "offset": 35740254208, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 35740286976, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.63.ffn_norm.weight", "offset": 35891675136, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 35891707904, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 35891740672, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.64.ffn_gate.weight", "offset": 36043128832, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.64.ffn_up.weight", "offset": 36194516992, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.64.ffn_norm.weight", "offset": 36345905152, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 36345937920, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 36345942016, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.64.attn_output.weight", "offset": 36351184896, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.64.attn_q.bias", "offset": 36393127936, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 36393160704, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.64.attn_v.bias", "offset": 36435103744, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 36435107840, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.65.attn_norm.weight", "offset": 36440350720, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 36440383488, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.65.ffn_gate.weight", "offset": 36591771648, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.65.ffn_up.weight", "offset": 36743159808, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.65.ffn_norm.weight", "offset": 36894547968, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 36894580736, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 36894584832, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.65.attn_output.weight", "offset": 36899827712, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.65.attn_q.bias", "offset": 36941770752, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 36941803520, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.65.attn_v.bias", "offset": 36983746560, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 36983750656, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.66.attn_norm.weight", "offset": 36988993536, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 36989026304, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.66.ffn_gate.weight", "offset": 37140414464, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.66.ffn_up.weight", "offset": 37291802624, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.66.ffn_norm.weight", "offset": 37443190784, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 37443223552, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 37443227648, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.66.attn_output.weight", "offset": 37448470528, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.66.attn_q.bias", "offset": 37490413568, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 37490446336, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.66.attn_v.bias", "offset": 37532389376, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 37532393472, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.67.attn_norm.weight", "offset": 37537636352, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 37537669120, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.67.ffn_gate.weight", "offset": 37689057280, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.67.ffn_up.weight", "offset": 37840445440, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.67.ffn_norm.weight", "offset": 37991833600, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 37991866368, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 37991870464, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.67.attn_output.weight", "offset": 37997113344, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.67.attn_q.bias", "offset": 38039056384, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 38039089152, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.67.attn_v.bias", "offset": 38081032192, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 38081036288, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.68.attn_k.bias", "offset": 38086279168, "shape": [ 1024 ], "size": 5242880, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 38086283264, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.68.attn_output.weight", "offset": 38091526144, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.68.attn_q.bias", "offset": 38133469184, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 38133501952, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.68.attn_v.bias", "offset": 38175444992, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 38175449088, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.68.attn_norm.weight", "offset": 38180691968, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 38180724736, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.68.ffn_gate.weight", "offset": 38332112896, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.68.ffn_up.weight", "offset": 38483501056, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.68.ffn_norm.weight", "offset": 38634889216, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 38634921984, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 38634954752, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.69.ffn_gate.weight", "offset": 38786342912, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.69.ffn_up.weight", "offset": 38937731072, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.69.ffn_norm.weight", "offset": 39089119232, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 39089152000, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 39089156096, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.69.attn_output.weight", "offset": 39094398976, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.69.attn_q.bias", "offset": 39136342016, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 39136374784, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.69.attn_v.bias", "offset": 39178317824, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 39178321920, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.70.ffn_gate.weight", "offset": 39183564800, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.70.attn_k.bias", "offset": 39334952960, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 39334957056, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.70.attn_output.weight", "offset": 39340199936, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.70.attn_q.bias", "offset": 39382142976, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 39382175744, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.70.attn_v.bias", "offset": 39424118784, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 39424122880, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.70.attn_norm.weight", "offset": 39429365760, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 39429398528, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.70.ffn_up.weight", "offset": 39580786688, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.70.ffn_norm.weight", "offset": 39732174848, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 39732207616, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 39732240384, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.71.ffn_gate.weight", "offset": 39883628544, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.71.ffn_up.weight", "offset": 40035016704, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.71.ffn_norm.weight", "offset": 40186404864, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 40186437632, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 40186441728, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.71.attn_output.weight", "offset": 40191684608, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.71.attn_q.bias", "offset": 40233627648, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 40233660416, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.71.attn_v.bias", "offset": 40275603456, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 40275607552, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.72.ffn_gate.weight", "offset": 40280850432, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.72.ffn_up.weight", "offset": 40432238592, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.72.attn_k.bias", "offset": 40583626752, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 40583630848, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.72.attn_output.weight", "offset": 40588873728, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.72.attn_q.bias", "offset": 40630816768, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 40630849536, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.72.attn_v.bias", "offset": 40672792576, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 40672796672, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.72.attn_norm.weight", "offset": 40678039552, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 40678072320, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.72.ffn_norm.weight", "offset": 40829460480, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 40829493248, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 40829526016, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.73.ffn_gate.weight", "offset": 40980914176, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.73.ffn_up.weight", "offset": 41132302336, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.73.ffn_norm.weight", "offset": 41283690496, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 41283723264, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 41283727360, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.73.attn_output.weight", "offset": 41288970240, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.73.attn_q.bias", "offset": 41330913280, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 41330946048, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.73.attn_v.bias", "offset": 41372889088, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 41372893184, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.74.attn_norm.weight", "offset": 41378136064, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 41378168832, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.74.ffn_gate.weight", "offset": 41529556992, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.74.ffn_up.weight", "offset": 41680945152, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.74.ffn_norm.weight", "offset": 41832333312, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 41832366080, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 41832370176, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.74.attn_output.weight", "offset": 41837613056, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.74.attn_q.bias", "offset": 41879556096, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 41879588864, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.74.attn_v.bias", "offset": 41921531904, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 41921536000, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.75.attn_norm.weight", "offset": 41926778880, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 41926811648, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.75.ffn_gate.weight", "offset": 42078199808, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.75.ffn_up.weight", "offset": 42229587968, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.75.ffn_norm.weight", "offset": 42380976128, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 42381008896, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 42381012992, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.75.attn_output.weight", "offset": 42386255872, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.75.attn_q.bias", "offset": 42428198912, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 42428231680, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.75.attn_v.bias", "offset": 42470174720, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 42470178816, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.76.attn_norm.weight", "offset": 42475421696, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 42475454464, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.76.ffn_gate.weight", "offset": 42626842624, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.76.ffn_up.weight", "offset": 42778230784, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.76.ffn_norm.weight", "offset": 42929618944, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 42929651712, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 42929655808, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.76.attn_output.weight", "offset": 42934898688, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.76.attn_q.bias", "offset": 42976841728, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 42976874496, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.76.attn_v.bias", "offset": 43018817536, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 43018821632, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.77.attn_k.bias", "offset": 43024064512, "shape": [ 1024 ], "size": 5242880, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 43024068608, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.77.attn_output.weight", "offset": 43029311488, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.77.attn_q.bias", "offset": 43071254528, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 43071287296, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.77.attn_v.bias", "offset": 43113230336, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 43113234432, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.77.attn_norm.weight", "offset": 43118477312, "shape": [ 8192 ], "size": 5242880, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 43118510080, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.77.ffn_gate.weight", "offset": 43269898240, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.77.ffn_up.weight", "offset": 43421286400, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.77.ffn_norm.weight", "offset": 43572674560, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 43572707328, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 43572740096, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.78.ffn_gate.weight", "offset": 43724128256, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.78.ffn_up.weight", "offset": 43875516416, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.78.ffn_norm.weight", "offset": 44026904576, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 44026937344, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 44026941440, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.78.attn_output.weight", "offset": 44032184320, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.78.attn_q.bias", "offset": 44074127360, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 44074160128, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.78.attn_v.bias", "offset": 44116103168, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 44116107264, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.79.ffn_gate.weight", "offset": 44121350144, "shape": [ 8192, 29568 ], "size": 5242880, "type": 3 }, { "name": "blk.79.attn_k.bias", "offset": 44272738304, "shape": [ 1024 ], "size": 151388160, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 44272742400, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "blk.79.attn_output.weight", "offset": 44277985280, "shape": [ 8192, 8192 ], "size": 5242880, "type": 3 }, { "name": "blk.79.attn_q.bias", "offset": 44319928320, "shape": [ 8192 ], "size": 41943040, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 44319961088, "shape": [ 8192, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.79.attn_v.bias", "offset": 44361904128, "shape": [ 1024 ], "size": 41943040, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 44361908224, "shape": [ 8192, 1024 ], "size": 4096, "type": 3 }, { "name": "output.weight", "offset": 44367151104, "shape": [ 8192, 152064 ], "size": 5242880, "type": 14 }, { "name": "blk.79.attn_norm.weight", "offset": 45389021184, "shape": [ 8192 ], "size": 1021870080, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 45389053952, "shape": [ 29568, 8192 ], "size": 32768, "type": 3 }, { "name": "blk.79.ffn_up.weight", "offset": 45540442112, "shape": [ 8192, 29568 ], "size": 151388160, "type": 3 }, { "name": "blk.79.ffn_norm.weight", "offset": 45691830272, "shape": [ 8192 ], "size": 151388160, "type": 0 }, { "name": "output_norm.weight", "offset": 45691863040, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }