qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 12, "general.name": "Qwen2-72B", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.00001, "qwen2.block_count": 80, "qwen2.context_length": 131072, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 535265280, "shape": [ 8192, 29568 ], "size": 535265280, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 639344640, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.0.attn_k.bias", "offset": 743424000, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 743428096, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 747032576, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.0.attn_q.bias", "offset": 784781312, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 784814080, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.0.attn_v.bias", "offset": 813649920, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 813654016, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 819421184, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 819453952, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 1001119744, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1001152512, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1001185280, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.1.ffn_gate.weight", "offset": 1182851072, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 1286930432, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 1391009792, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 1391042560, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1391046656, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 1394651136, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.1.attn_q.bias", "offset": 1432399872, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 1432432640, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.1.attn_v.bias", "offset": 1461268480, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 1461272576, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 1467039744, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1467072512, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.2.ffn_gate.weight", "offset": 1648738304, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 1752817664, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 1856897024, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 1856929792, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1856933888, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 1860538368, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.2.attn_q.bias", "offset": 1898287104, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 1898319872, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.2.attn_v.bias", "offset": 1927155712, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 1927159808, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 1932926976, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1932959744, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.3.ffn_gate.weight", "offset": 2114625536, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 2218704896, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 2322784256, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 2322817024, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 2322821120, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 2326425600, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.3.attn_q.bias", "offset": 2364174336, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 2364207104, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.3.attn_v.bias", "offset": 2393042944, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 2393047040, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 2398814208, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 2398846976, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.4.ffn_gate.weight", "offset": 2580512768, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 2684592128, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 2788671488, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 2788704256, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 2788708352, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 2792312832, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.4.attn_q.bias", "offset": 2830061568, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 2830094336, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.4.attn_v.bias", "offset": 2858930176, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 2858934272, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_k.bias", "offset": 2864701440, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 2864705536, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 2868310016, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.5.attn_q.bias", "offset": 2906058752, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 2906091520, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.5.attn_v.bias", "offset": 2934927360, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 2934931456, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 2940698624, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 2940731392, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.5.ffn_gate.weight", "offset": 3107258368, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 3211337728, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 3315417088, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 3315449856, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 3315482624, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.6.ffn_gate.weight", "offset": 3482009600, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 3586088960, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 3690168320, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 3690201088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 3690205184, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 3693809664, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.6.attn_q.bias", "offset": 3731558400, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 3731591168, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.6.attn_v.bias", "offset": 3760427008, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 3760431104, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 3766198272, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.7.attn_k.bias", "offset": 3870277632, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 3870281728, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 3873886208, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.7.attn_q.bias", "offset": 3911634944, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 3911667712, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.7.attn_v.bias", "offset": 3940503552, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 3940507648, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 3946274816, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 3946307584, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.7.ffn_up.weight", "offset": 4112834560, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 4216913920, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 4216946688, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 4216979456, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.8.ffn_gate.weight", "offset": 4383506432, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 4487585792, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 4591665152, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 4591697920, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 4591702016, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 4595306496, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.8.attn_q.bias", "offset": 4633055232, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 4633088000, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.8.attn_v.bias", "offset": 4661923840, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 4661927936, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 4667695104, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 4771774464, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.9.attn_k.bias", "offset": 4875853824, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 4875857920, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 4879462400, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.9.attn_q.bias", "offset": 4917211136, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 4917243904, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.9.attn_v.bias", "offset": 4946079744, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 4946083840, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 4951851008, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 4951883776, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.10.ffn_gate.weight", "offset": 5118410752, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 5222490112, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 5326569472, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 5326602240, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 5326606336, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 5330210816, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.10.attn_q.bias", "offset": 5367959552, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 5367992320, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.10.attn_v.bias", "offset": 5396828160, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 5396832256, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 5402599424, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 5402632192, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.11.ffn_gate.weight", "offset": 5569159168, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 5673238528, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 5777317888, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 5777350656, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 5777354752, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 5780959232, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.11.attn_q.bias", "offset": 5818707968, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 5818740736, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.11.attn_v.bias", "offset": 5847576576, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 5847580672, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 5853347840, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 5853380608, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.9.ffn_norm.weight", "offset": 6019907584, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 6019940352, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 6019973120, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.12.ffn_gate.weight", "offset": 6186500096, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 6290579456, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 6394658816, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 6394691584, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 6394695680, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 6398300160, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.12.attn_q.bias", "offset": 6436048896, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 6436081664, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.12.attn_v.bias", "offset": 6464917504, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 6464921600, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 6470688768, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 6470721536, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.13.ffn_gate.weight", "offset": 6637248512, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 6741327872, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 6845407232, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 6845440000, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 6845444096, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 6849048576, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.13.attn_q.bias", "offset": 6886797312, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 6886830080, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.13.attn_v.bias", "offset": 6915665920, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 6915670016, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_k.bias", "offset": 6921437184, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 6921441280, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 6925045760, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.14.attn_q.bias", "offset": 6962794496, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 6962827264, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.14.attn_v.bias", "offset": 6991663104, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 6991667200, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 6997434368, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 6997467136, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.14.ffn_gate.weight", "offset": 7163994112, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 7268073472, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 7372152832, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 7372185600, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 7372218368, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.15.ffn_gate.weight", "offset": 7538745344, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 7642824704, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 7746904064, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 7746936832, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 7746940928, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 7750545408, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.15.attn_q.bias", "offset": 7788294144, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 7788326912, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.15.attn_v.bias", "offset": 7817162752, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 7817166848, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 7822934016, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.16.attn_k.bias", "offset": 7927013376, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 7927017472, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 7930621952, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.16.attn_q.bias", "offset": 7968370688, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 7968403456, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.16.attn_v.bias", "offset": 7997239296, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 7997243392, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 8003010560, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 8003043328, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.16.ffn_up.weight", "offset": 8169570304, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 8273649664, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 8273682432, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 8273715200, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.17.ffn_gate.weight", "offset": 8440242176, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 8544321536, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 8648400896, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 8648433664, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 8648437760, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 8652042240, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.17.attn_q.bias", "offset": 8689790976, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 8689823744, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.17.attn_v.bias", "offset": 8718659584, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 8718663680, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 8724430848, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 8828510208, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.18.attn_k.bias", "offset": 8932589568, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 8932593664, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 8936198144, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.18.attn_q.bias", "offset": 8973946880, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 8973979648, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.18.attn_v.bias", "offset": 9002815488, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 9002819584, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 9008586752, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 9008619520, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.18.ffn_norm.weight", "offset": 9175146496, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 9175179264, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 9175212032, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.19.ffn_gate.weight", "offset": 9341739008, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 9445818368, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 9549897728, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 9549930496, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 9549934592, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 9553539072, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.19.attn_q.bias", "offset": 9591287808, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 9591320576, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.19.attn_v.bias", "offset": 9620156416, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 9620160512, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 9625927680, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 9625960448, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.20.ffn_gate.weight", "offset": 9792487424, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 9896566784, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 10000646144, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 10000678912, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 10000683008, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 10004287488, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.20.attn_q.bias", "offset": 10042036224, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 10042068992, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.20.attn_v.bias", "offset": 10070904832, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 10070908928, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 10076676096, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 10076708864, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.21.ffn_gate.weight", "offset": 10243235840, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 10347315200, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 10451394560, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 10451427328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 10451431424, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 10455035904, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.21.attn_q.bias", "offset": 10492784640, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 10492817408, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.21.attn_v.bias", "offset": 10521653248, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 10521657344, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 10527424512, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 10527457280, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.22.ffn_gate.weight", "offset": 10693984256, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 10798063616, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 10902142976, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 10902175744, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 10902179840, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 10905784320, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.22.attn_q.bias", "offset": 10943533056, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 10943565824, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.22.attn_v.bias", "offset": 10972401664, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 10972405760, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_k.bias", "offset": 10978172928, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 10978177024, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 10981781504, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.23.attn_q.bias", "offset": 11019530240, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 11019563008, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.23.attn_v.bias", "offset": 11048398848, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 11048402944, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 11054170112, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 11054202880, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.23.ffn_gate.weight", "offset": 11220729856, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 11324809216, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 11428888576, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 11428921344, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 11428954112, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.24.ffn_gate.weight", "offset": 11595481088, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 11699560448, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 11803639808, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 11803672576, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 11803676672, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 11807281152, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.24.attn_q.bias", "offset": 11845029888, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 11845062656, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.24.attn_v.bias", "offset": 11873898496, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 11873902592, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 11879669760, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.25.attn_k.bias", "offset": 11983749120, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 11983753216, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 11987357696, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.25.attn_q.bias", "offset": 12025106432, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 12025139200, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.25.attn_v.bias", "offset": 12053975040, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 12053979136, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 12059746304, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 12059779072, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.25.ffn_up.weight", "offset": 12226306048, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 12330385408, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 12330418176, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 12330450944, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.26.ffn_gate.weight", "offset": 12496977920, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 12601057280, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 12705136640, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 12705169408, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 12705173504, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 12708777984, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.26.attn_q.bias", "offset": 12746526720, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 12746559488, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.26.attn_v.bias", "offset": 12775395328, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 12775399424, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 12781166592, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 12885245952, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.27.attn_k.bias", "offset": 12989325312, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 12989329408, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 12992933888, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.27.attn_q.bias", "offset": 13030682624, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 13030715392, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.27.attn_v.bias", "offset": 13059551232, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 13059555328, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 13065322496, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 13065355264, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.27.ffn_norm.weight", "offset": 13231882240, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 13231915008, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 13231947776, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.28.ffn_gate.weight", "offset": 13398474752, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 13502554112, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 13606633472, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 13606666240, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 13606670336, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 13610274816, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.28.attn_q.bias", "offset": 13648023552, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 13648056320, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.28.attn_v.bias", "offset": 13676892160, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 13676896256, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 13682663424, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 13682696192, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.29.ffn_gate.weight", "offset": 13849223168, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 13953302528, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 14057381888, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 14057414656, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 14057418752, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 14061023232, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.29.attn_q.bias", "offset": 14098771968, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 14098804736, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.29.attn_v.bias", "offset": 14127640576, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 14127644672, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 14133411840, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 14133444608, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.30.ffn_gate.weight", "offset": 14299971584, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 14404050944, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 14508130304, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 14508163072, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 14508167168, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 14511771648, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.30.attn_q.bias", "offset": 14549520384, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 14549553152, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.30.attn_v.bias", "offset": 14578388992, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 14578393088, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 14584160256, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 14584193024, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.31.ffn_gate.weight", "offset": 14750720000, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 14854799360, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 14958878720, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 14958911488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 14958915584, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.31.attn_output.weight", "offset": 14962520064, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.31.attn_q.bias", "offset": 15000268800, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 15000301568, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.31.attn_v.bias", "offset": 15029137408, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 15029141504, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_k.bias", "offset": 15034908672, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 15034912768, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 15038517248, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.32.attn_q.bias", "offset": 15076265984, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 15076298752, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.32.attn_v.bias", "offset": 15105134592, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 15105138688, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 15110905856, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 15110938624, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.32.ffn_gate.weight", "offset": 15277465600, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 15381544960, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 15485624320, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 15485657088, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 15485689856, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.33.ffn_gate.weight", "offset": 15652216832, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 15756296192, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 15860375552, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 15860408320, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 15860412416, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 15864016896, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.33.attn_q.bias", "offset": 15901765632, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 15901798400, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.33.attn_v.bias", "offset": 15930634240, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 15930638336, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.ffn_gate.weight", "offset": 15936405504, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.34.attn_k.bias", "offset": 16040484864, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 16040488960, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 16044093440, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.34.attn_q.bias", "offset": 16081842176, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 16081874944, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.34.attn_v.bias", "offset": 16110710784, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 16110714880, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 16116482048, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 16116514816, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.34.ffn_up.weight", "offset": 16283041792, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 16387121152, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 16387153920, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 16387186688, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.35.ffn_gate.weight", "offset": 16553713664, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 16657793024, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 16761872384, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 16761905152, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 16761909248, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 16765513728, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.35.attn_q.bias", "offset": 16803262464, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 16803295232, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.35.attn_v.bias", "offset": 16832131072, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 16832135168, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 16837902336, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 16941981696, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.36.attn_k.bias", "offset": 17046061056, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 17046065152, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 17049669632, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.36.attn_q.bias", "offset": 17087418368, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 17087451136, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.36.attn_v.bias", "offset": 17116286976, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 17116291072, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 17122058240, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 17122091008, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.36.ffn_norm.weight", "offset": 17288617984, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 17288650752, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 17288683520, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.37.ffn_gate.weight", "offset": 17455210496, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 17559289856, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 17663369216, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 17663401984, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 17663406080, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 17667010560, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.37.attn_q.bias", "offset": 17704759296, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 17704792064, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.37.attn_v.bias", "offset": 17733627904, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 17733632000, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 17739399168, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 17739431936, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.38.ffn_gate.weight", "offset": 17905958912, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 18010038272, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 18114117632, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 18114150400, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 18114154496, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 18117758976, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.38.attn_q.bias", "offset": 18155507712, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 18155540480, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.38.attn_v.bias", "offset": 18184376320, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 18184380416, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 18190147584, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 18190180352, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.39.ffn_gate.weight", "offset": 18356707328, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 18460786688, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 18564866048, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 18564898816, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 18564902912, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 18568507392, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.39.attn_q.bias", "offset": 18606256128, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 18606288896, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.39.attn_v.bias", "offset": 18635124736, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 18635128832, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 18640896000, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 18640928768, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.40.ffn_gate.weight", "offset": 18807455744, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.40.ffn_up.weight", "offset": 18911535104, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.40.ffn_norm.weight", "offset": 19015614464, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 19015647232, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 19015651328, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 19019255808, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.40.attn_q.bias", "offset": 19057004544, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 19057037312, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.40.attn_v.bias", "offset": 19085873152, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 19085877248, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_k.bias", "offset": 19091644416, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 19091648512, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 19095252992, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.41.attn_q.bias", "offset": 19133001728, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 19133034496, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.41.attn_v.bias", "offset": 19161870336, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 19161874432, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 19167641600, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 19167674368, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.41.ffn_gate.weight", "offset": 19334201344, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.41.ffn_up.weight", "offset": 19438280704, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.41.ffn_norm.weight", "offset": 19542360064, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 19542392832, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 19542425600, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.42.ffn_gate.weight", "offset": 19708952576, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.42.ffn_up.weight", "offset": 19813031936, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.42.ffn_norm.weight", "offset": 19917111296, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 19917144064, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 19917148160, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.42.attn_output.weight", "offset": 19920752640, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.42.attn_q.bias", "offset": 19958501376, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 19958534144, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.42.attn_v.bias", "offset": 19987369984, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 19987374080, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.ffn_gate.weight", "offset": 19993141248, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.43.attn_k.bias", "offset": 20097220608, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 20097224704, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.43.attn_output.weight", "offset": 20100829184, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.43.attn_q.bias", "offset": 20138577920, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 20138610688, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.43.attn_v.bias", "offset": 20167446528, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 20167450624, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 20173217792, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 20173250560, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.43.ffn_up.weight", "offset": 20339777536, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.43.ffn_norm.weight", "offset": 20443856896, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 20443889664, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 20443922432, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.44.ffn_gate.weight", "offset": 20610449408, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.44.ffn_up.weight", "offset": 20714528768, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.44.ffn_norm.weight", "offset": 20818608128, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 20818640896, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 20818644992, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.44.attn_output.weight", "offset": 20822249472, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.44.attn_q.bias", "offset": 20859998208, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 20860030976, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.44.attn_v.bias", "offset": 20888866816, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 20888870912, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.ffn_gate.weight", "offset": 20894638080, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.45.ffn_up.weight", "offset": 20998717440, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.45.attn_k.bias", "offset": 21102796800, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 21102800896, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.45.attn_output.weight", "offset": 21106405376, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.45.attn_q.bias", "offset": 21144154112, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 21144186880, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.45.attn_v.bias", "offset": 21173022720, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 21173026816, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 21178793984, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 21178826752, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.45.ffn_norm.weight", "offset": 21345353728, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 21345386496, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 21345419264, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.46.ffn_gate.weight", "offset": 21511946240, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.46.ffn_up.weight", "offset": 21616025600, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.46.ffn_norm.weight", "offset": 21720104960, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 21720137728, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 21720141824, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.46.attn_output.weight", "offset": 21723746304, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "lk.58.attn_output.", "offset": 21761495040, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 21761527808, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.46.attn_v.bias", "offset": 21790363648, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 21790367744, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 21796134912, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 21796167680, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.47.ffn_gate.weight", "offset": 21962694656, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.47.ffn_up.weight", "offset": 22066774016, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.47.ffn_norm.weight", "offset": 22170853376, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 22170886144, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 22170890240, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.47.attn_output.weight", "offset": 22174494720, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.47.attn_q.bias", "offset": 22212243456, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 22212276224, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.47.attn_v.bias", "offset": 22241112064, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 22241116160, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.48.attn_norm.weight", "offset": 22246883328, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 22246916096, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.48.ffn_gate.weight", "offset": 22413443072, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.48.ffn_up.weight", "offset": 22517522432, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.48.ffn_norm.weight", "offset": 22621601792, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 22621634560, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 22621638656, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.48.attn_output.weight", "offset": 22625243136, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.48.attn_q.bias", "offset": 22662991872, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 22663024640, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.48.attn_v.bias", "offset": 22691860480, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 22691864576, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.49.attn_norm.weight", "offset": 22697631744, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 22697664512, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.49.ffn_gate.weight", "offset": 22864191488, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.49.ffn_up.weight", "offset": 22968270848, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.49.ffn_norm.weight", "offset": 23072350208, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 23072382976, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 23072387072, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.49.attn_output.weight", "offset": 23075991552, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.49.attn_q.bias", "offset": 23113740288, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 23113773056, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.49.attn_v.bias", "offset": 23142608896, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 23142612992, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_k.bias", "offset": 23148380160, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 23148384256, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.50.attn_output.weight", "offset": 23151988736, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.50.attn_q.bias", "offset": 23189737472, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 23189770240, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.50.attn_v.bias", "offset": 23218606080, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 23218610176, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_norm.weight", "offset": 23224377344, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 23224410112, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.50.ffn_gate.weight", "offset": 23390937088, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.50.ffn_up.weight", "offset": 23495016448, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.50.ffn_norm.weight", "offset": 23599095808, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 23599128576, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 23599161344, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.51.ffn_gate.weight", "offset": 23765688320, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.51.ffn_up.weight", "offset": 23869767680, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.51.ffn_norm.weight", "offset": 23973847040, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 23973879808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 23973883904, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.51.attn_output.weight", "offset": 23977488384, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.51.attn_q.bias", "offset": 24015237120, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 24015269888, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.51.attn_v.bias", "offset": 24044105728, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 24044109824, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.ffn_gate.weight", "offset": 24049876992, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.52.attn_k.bias", "offset": 24153956352, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 24153960448, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.52.attn_output.weight", "offset": 24157564928, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.52.attn_q.bias", "offset": 24195313664, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 24195346432, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.52.attn_v.bias", "offset": 24224182272, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 24224186368, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.attn_norm.weight", "offset": 24229953536, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 24229986304, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.52.ffn_up.weight", "offset": 24396513280, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.52.ffn_norm.weight", "offset": 24500592640, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 24500625408, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 24500658176, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.53.ffn_gate.weight", "offset": 24667185152, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.53.ffn_up.weight", "offset": 24771264512, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.53.ffn_norm.weight", "offset": 24875343872, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 24875376640, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 24875380736, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.53.attn_output.weight", "offset": 24878985216, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.53.attn_q.bias", "offset": 24916733952, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 24916766720, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.53.attn_v.bias", "offset": 24945602560, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 24945606656, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.ffn_gate.weight", "offset": 24951373824, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.54.ffn_up.weight", "offset": 25055453184, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.54.attn_k.bias", "offset": 25159532544, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 25159536640, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.54.attn_output.weight", "offset": 25163141120, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.54.attn_q.bias", "offset": 25200889856, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 25200922624, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.54.attn_v.bias", "offset": 25229758464, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 25229762560, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.attn_norm.weight", "offset": 25235529728, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 25235562496, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.54.ffn_norm.weight", "offset": 25402089472, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 25402122240, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 25402155008, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.55.ffn_gate.weight", "offset": 25568681984, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.55.ffn_up.weight", "offset": 25672761344, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.55.ffn_norm.weight", "offset": 25776840704, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 25776873472, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 25776877568, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.55.attn_output.weight", "offset": 25780482048, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.55.attn_q.bias", "offset": 25818230784, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 25818263552, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.55.attn_v.bias", "offset": 25847099392, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 25847103488, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.56.attn_norm.weight", "offset": 25852870656, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 25852903424, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.56.ffn_gate.weight", "offset": 26019430400, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.56.ffn_up.weight", "offset": 26123509760, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.56.ffn_norm.weight", "offset": 26227589120, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 26227621888, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 26227625984, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.56.attn_output.weight", "offset": 26231230464, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.56.attn_q.bias", "offset": 26268979200, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 26269011968, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.56.attn_v.bias", "offset": 26297847808, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 26297851904, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.57.attn_norm.weight", "offset": 26303619072, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 26303651840, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.57.ffn_gate.weight", "offset": 26470178816, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.57.ffn_up.weight", "offset": 26574258176, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.57.ffn_norm.weight", "offset": 26678337536, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 26678370304, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 26678374400, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.57.attn_output.weight", "offset": 26681978880, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.57.attn_q.bias", "offset": 26719727616, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 26719760384, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.57.attn_v.bias", "offset": 26748596224, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 26748600320, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.58.attn_norm.weight", "offset": 26754367488, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 26754400256, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.58.ffn_gate.weight", "offset": 26920927232, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.58.ffn_up.weight", "offset": 27025006592, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.58.ffn_norm.weight", "offset": 27129085952, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 27129118720, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 27129122816, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.58.attn_output.weight", "offset": 27132727296, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.58.attn_q.bias", "offset": 27170476032, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 27170508800, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.58.attn_v.bias", "offset": 27199344640, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 27199348736, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_k.bias", "offset": 27205115904, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 27205120000, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.59.attn_output.weight", "offset": 27208724480, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.59.attn_q.bias", "offset": 27246473216, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 27246505984, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.59.attn_v.bias", "offset": 27275341824, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 27275345920, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_norm.weight", "offset": 27281113088, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 27281145856, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.59.ffn_gate.weight", "offset": 27447672832, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.59.ffn_up.weight", "offset": 27551752192, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.59.ffn_norm.weight", "offset": 27655831552, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 27655864320, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 27655897088, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.60.ffn_gate.weight", "offset": 27822424064, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.60.ffn_up.weight", "offset": 27926503424, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.60.ffn_norm.weight", "offset": 28030582784, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 28030615552, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 28030619648, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.60.attn_output.weight", "offset": 28034224128, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.60.attn_q.bias", "offset": 28071972864, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 28072005632, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.60.attn_v.bias", "offset": 28100841472, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 28100845568, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.ffn_gate.weight", "offset": 28106612736, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.61.attn_k.bias", "offset": 28210692096, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 28210696192, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.61.attn_output.weight", "offset": 28214300672, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.61.attn_q.bias", "offset": 28252049408, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 28252082176, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.61.attn_v.bias", "offset": 28280918016, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 28280922112, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.attn_norm.weight", "offset": 28286689280, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 28286722048, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.61.ffn_up.weight", "offset": 28453249024, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.61.ffn_norm.weight", "offset": 28557328384, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 28557361152, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 28557393920, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.62.ffn_gate.weight", "offset": 28723920896, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.62.ffn_up.weight", "offset": 28828000256, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.62.ffn_norm.weight", "offset": 28932079616, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 28932112384, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 28932116480, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.62.attn_output.weight", "offset": 28935720960, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.62.attn_q.bias", "offset": 28973469696, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 28973502464, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.62.attn_v.bias", "offset": 29002338304, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 29002342400, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.ffn_gate.weight", "offset": 29008109568, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.63.ffn_up.weight", "offset": 29112188928, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.63.attn_k.bias", "offset": 29216268288, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 29216272384, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.63.attn_output.weight", "offset": 29219876864, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.63.attn_q.bias", "offset": 29257625600, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 29257658368, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.63.attn_v.bias", "offset": 29286494208, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 29286498304, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.attn_norm.weight", "offset": 29292265472, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 29292298240, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.63.ffn_norm.weight", "offset": 29458825216, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 29458857984, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 29458890752, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.64.ffn_gate.weight", "offset": 29625417728, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.64.ffn_up.weight", "offset": 29729497088, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.64.ffn_norm.weight", "offset": 29833576448, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 29833609216, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 29833613312, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.64.attn_output.weight", "offset": 29837217792, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.64.attn_q.bias", "offset": 29874966528, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 29874999296, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.64.attn_v.bias", "offset": 29903835136, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 29903839232, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.65.attn_norm.weight", "offset": 29909606400, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 29909639168, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.65.ffn_gate.weight", "offset": 30076166144, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.65.ffn_up.weight", "offset": 30180245504, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.65.ffn_norm.weight", "offset": 30284324864, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 30284357632, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 30284361728, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.65.attn_output.weight", "offset": 30287966208, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.65.attn_q.bias", "offset": 30325714944, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 30325747712, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.65.attn_v.bias", "offset": 30354583552, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 30354587648, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.66.attn_norm.weight", "offset": 30360354816, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 30360387584, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.66.ffn_gate.weight", "offset": 30526914560, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.66.ffn_up.weight", "offset": 30630993920, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.66.ffn_norm.weight", "offset": 30735073280, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 30735106048, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 30735110144, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.66.attn_output.weight", "offset": 30738714624, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.66.attn_q.bias", "offset": 30776463360, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 30776496128, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.66.attn_v.bias", "offset": 30805331968, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 30805336064, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.67.attn_norm.weight", "offset": 30811103232, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 30811136000, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.67.ffn_gate.weight", "offset": 30977662976, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.67.ffn_up.weight", "offset": 31081742336, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.67.ffn_norm.weight", "offset": 31185821696, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 31185854464, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 31185858560, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.67.attn_output.weight", "offset": 31189463040, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.67.attn_q.bias", "offset": 31227211776, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 31227244544, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.67.attn_v.bias", "offset": 31256080384, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 31256084480, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_k.bias", "offset": 31261851648, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 31261855744, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.68.attn_output.weight", "offset": 31265460224, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.68.attn_q.bias", "offset": 31303208960, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 31303241728, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.68.attn_v.bias", "offset": 31332077568, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 31332081664, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_norm.weight", "offset": 31337848832, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 31337881600, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.68.ffn_gate.weight", "offset": 31504408576, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.68.ffn_up.weight", "offset": 31608487936, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.68.ffn_norm.weight", "offset": 31712567296, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 31712600064, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 31712632832, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.69.ffn_gate.weight", "offset": 31879159808, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.69.ffn_up.weight", "offset": 31983239168, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.69.ffn_norm.weight", "offset": 32087318528, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 32087351296, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 32087355392, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.69.attn_output.weight", "offset": 32090959872, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.69.attn_q.bias", "offset": 32128708608, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 32128741376, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.69.attn_v.bias", "offset": 32157577216, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 32157581312, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.ffn_gate.weight", "offset": 32163348480, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.70.attn_k.bias", "offset": 32267427840, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 32267431936, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.70.attn_output.weight", "offset": 32271036416, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.70.attn_q.bias", "offset": 32308785152, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 32308817920, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "\u0014\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.70.att", "offset": 32337653760, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 32337657856, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.attn_norm.weight", "offset": 32343425024, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 32343457792, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.70.ffn_up.weight", "offset": 32509984768, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.70.ffn_norm.weight", "offset": 32614064128, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 32614096896, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 32614129664, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.71.ffn_gate.weight", "offset": 32780656640, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.71.ffn_up.weight", "offset": 32884736000, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.71.ffn_norm.weight", "offset": 32988815360, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 32988848128, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 32988852224, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.71.attn_output.weight", "offset": 32992456704, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.71.attn_q.bias", "offset": 33030205440, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 33030238208, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.71.attn_v.bias", "offset": 33059074048, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 33059078144, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.ffn_gate.weight", "offset": 33064845312, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.72.ffn_up.weight", "offset": 33168924672, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.72.attn_k.bias", "offset": 33273004032, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 33273008128, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.72.attn_output.weight", "offset": 33276612608, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.72.attn_q.bias", "offset": 33314361344, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 33314394112, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.72.attn_v.bias", "offset": 33343229952, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 33343234048, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.attn_norm.weight", "offset": 33349001216, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 33349033984, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.72.ffn_norm.weight", "offset": 33515560960, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 33515593728, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 33515626496, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.73.ffn_gate.weight", "offset": 33682153472, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.73.ffn_up.weight", "offset": 33786232832, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.73.ffn_norm.weight", "offset": 33890312192, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 33890344960, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 33890349056, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.73.attn_output.weight", "offset": 33893953536, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.73.attn_q.bias", "offset": 33931702272, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 33931735040, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.73.attn_v.bias", "offset": 33960570880, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 33960574976, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.74.attn_norm.weight", "offset": 33966342144, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 33966374912, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.74.ffn_gate.weight", "offset": 34132901888, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.74.ffn_up.weight", "offset": 34236981248, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.74.ffn_norm.weight", "offset": 34341060608, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 34341093376, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 34341097472, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.74.attn_output.weight", "offset": 34344701952, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.74.attn_q.bias", "offset": 34382450688, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 34382483456, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.74.attn_v.bias", "offset": 34411319296, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 34411323392, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.75.attn_norm.weight", "offset": 34417090560, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 34417123328, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.75.ffn_gate.weight", "offset": 34583650304, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.75.ffn_up.weight", "offset": 34687729664, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.75.ffn_norm.weight", "offset": 34791809024, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 34791841792, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 34791845888, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.75.attn_output.weight", "offset": 34795450368, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.75.attn_q.bias", "offset": 34833199104, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 34833231872, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.75.attn_v.bias", "offset": 34862067712, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 34862071808, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.76.attn_norm.weight", "offset": 34867838976, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 34867871744, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.76.ffn_gate.weight", "offset": 35034398720, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.76.ffn_up.weight", "offset": 35138478080, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.76.ffn_norm.weight", "offset": 35242557440, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 35242590208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 35242594304, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.76.attn_output.weight", "offset": 35246198784, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.76.attn_q.bias", "offset": 35283947520, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 35283980288, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.76.attn_v.bias", "offset": 35312816128, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 35312820224, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_k.bias", "offset": 35318587392, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 35318591488, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.77.attn_output.weight", "offset": 35322195968, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.77.attn_q.bias", "offset": 35359944704, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 35359977472, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.77.attn_v.bias", "offset": 35388813312, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 35388817408, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_norm.weight", "offset": 35394584576, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 35394617344, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.77.ffn_gate.weight", "offset": 35561144320, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.77.ffn_up.weight", "offset": 35665223680, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.77.ffn_norm.weight", "offset": 35769303040, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 35769335808, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 35769368576, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.78.ffn_gate.weight", "offset": 35935895552, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.78.ffn_up.weight", "offset": 36039974912, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.78.ffn_norm.weight", "offset": 36144054272, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 36144087040, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 36144091136, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.78.attn_output.weight", "offset": 36147695616, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.78.attn_q.bias", "offset": 36185444352, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 36185477120, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.78.attn_v.bias", "offset": 36214312960, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 36214317056, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.79.ffn_gate.weight", "offset": 36220084224, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.79.attn_k.bias", "offset": 36324163584, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 36324167680, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.79.attn_output.weight", "offset": 36327772160, "shape": [ 8192, 8192 ], "size": 3604480, "type": 12 }, { "name": "blk.79.attn_q.bias", "offset": 36365520896, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 36365553664, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.79.attn_v.bias", "offset": 36394389504, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 36394393600, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "output.weight", "offset": 36400160768, "shape": [ 8192, 152064 ], "size": 5767168, "type": 14 }, { "name": "blk.79.attn_norm.weight", "offset": 37422030848, "shape": [ 8192 ], "size": 1021870080, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 37422063616, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.79.ffn_up.weight", "offset": 37588590592, "shape": [ 8192, 29568 ], "size": 166526976, "type": 11 }, { "name": "blk.79.ffn_norm.weight", "offset": 37692669952, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "output_norm.weight", "offset": 37692702720, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }