qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 18, "general.name": "Qwen2-72B", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.00001, "qwen2.block_count": 80, "qwen2.context_length": 131072, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 1021870080, "shape": [ 8192, 29568 ], "size": 1021870080, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 1220567040, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.0.attn_k.bias", "offset": 1419264000, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 1419268096, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.0.attn_output.weight", "offset": 1426149376, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.0.attn_q.bias", "offset": 1481199616, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 1481232384, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.0.attn_v.bias", "offset": 1536282624, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 1536286720, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.0.attn_norm.weight", "offset": 1543168000, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 1543200768, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 1800560640, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1800593408, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1800626176, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.1.ffn_gate.weight", "offset": 2057986048, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 2256683008, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.1.ffn_norm.weight", "offset": 2455379968, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 2455412736, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 2455416832, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.1.attn_output.weight", "offset": 2462298112, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.1.attn_q.bias", "offset": 2517348352, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 2517381120, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.1.attn_v.bias", "offset": 2572431360, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 2572435456, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 2579316736, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 2579349504, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.2.ffn_gate.weight", "offset": 2836709376, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 3035406336, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.2.ffn_norm.weight", "offset": 3234103296, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 3234136064, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 3234140160, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.2.attn_output.weight", "offset": 3241021440, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.2.attn_q.bias", "offset": 3296071680, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 3296104448, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.2.attn_v.bias", "offset": 3351154688, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 3351158784, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 3358040064, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 3358072832, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.3.ffn_gate.weight", "offset": 3615432704, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.3.ffn_up.weight", "offset": 3814129664, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.3.ffn_norm.weight", "offset": 4012826624, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 4012859392, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 4012863488, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.3.attn_output.weight", "offset": 4019744768, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.3.attn_q.bias", "offset": 4074795008, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 4074827776, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.3.attn_v.bias", "offset": 4129878016, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 4129882112, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 4136763392, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 4136796160, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.4.ffn_gate.weight", "offset": 4394156032, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 4592852992, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.4.ffn_norm.weight", "offset": 4791549952, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 4791582720, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 4791586816, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.4.attn_output.weight", "offset": 4798468096, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.4.attn_q.bias", "offset": 4853518336, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 4853551104, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.4.attn_v.bias", "offset": 4908601344, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 4908605440, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.5.attn_k.bias", "offset": 4915486720, "shape": [ 1024 ], "size": 6881280, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 4915490816, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.5.attn_output.weight", "offset": 4922372096, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.5.attn_q.bias", "offset": 4977422336, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 4977455104, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.5.attn_v.bias", "offset": 5032505344, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 5032509440, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 5039390720, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 5039423488, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.5.ffn_gate.weight", "offset": 5296783360, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.5.ffn_up.weight", "offset": 5495480320, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.5.ffn_norm.weight", "offset": 5694177280, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 5694210048, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 5694242816, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.6.ffn_gate.weight", "offset": 5951602688, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.6.ffn_up.weight", "offset": 6150299648, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.6.ffn_norm.weight", "offset": 6348996608, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 6349029376, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 6349033472, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.6.attn_output.weight", "offset": 6355914752, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.6.attn_q.bias", "offset": 6410964992, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 6410997760, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.6.attn_v.bias", "offset": 6466048000, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 6466052096, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 6472933376, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.7.attn_k.bias", "offset": 6671630336, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 6671634432, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.7.attn_output.weight", "offset": 6678515712, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.7.attn_q.bias", "offset": 6733565952, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 6733598720, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.7.attn_v.bias", "offset": 6788648960, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 6788653056, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 6795534336, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 6795567104, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 7052926976, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.7.ffn_norm.weight", "offset": 7251623936, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 7251656704, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 7251689472, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.8.ffn_gate.weight", "offset": 7509049344, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.8.ffn_up.weight", "offset": 7707746304, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.8.ffn_norm.weight", "offset": 7906443264, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 7906476032, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 7906480128, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.8.attn_output.weight", "offset": 7913361408, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.8.attn_q.bias", "offset": 7968411648, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 7968444416, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.8.attn_v.bias", "offset": 8023494656, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 8023498752, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 8030380032, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.9.ffn_up.weight", "offset": 8229076992, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.9.attn_k.bias", "offset": 8427773952, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 8427778048, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.9.attn_output.weight", "offset": 8434659328, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.9.attn_q.bias", "offset": 8489709568, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 8489742336, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.9.attn_v.bias", "offset": 8544792576, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 8544796672, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 8551677952, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 8551710720, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.10.ffn_gate.weight", "offset": 8809070592, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.10.ffn_up.weight", "offset": 9007767552, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.10.ffn_norm.weight", "offset": 9206464512, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 9206497280, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 9206501376, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.10.attn_output.weight", "offset": 9213382656, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.10.attn_q.bias", "offset": 9268432896, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 9268465664, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.10.attn_v.bias", "offset": 9323515904, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 9323520000, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 9330401280, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 9330434048, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.11.ffn_gate.weight", "offset": 9587793920, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.11.ffn_up.weight", "offset": 9786490880, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.11.ffn_norm.weight", "offset": 9985187840, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 9985220608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 9985224704, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.11.attn_output.weight", "offset": 9992105984, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.11.attn_q.bias", "offset": 10047156224, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 10047188992, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.11.attn_v.bias", "offset": 10102239232, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 10102243328, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 10109124608, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 10109157376, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.9.ffn_norm.weight", "offset": 10366517248, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 10366550016, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 10366582784, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.12.ffn_gate.weight", "offset": 10623942656, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.12.ffn_up.weight", "offset": 10822639616, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.12.ffn_norm.weight", "offset": 11021336576, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 11021369344, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 11021373440, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.12.attn_output.weight", "offset": 11028254720, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.12.attn_q.bias", "offset": 11083304960, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 11083337728, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.12.attn_v.bias", "offset": 11138387968, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 11138392064, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 11145273344, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 11145306112, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.13.ffn_gate.weight", "offset": 11402665984, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.13.ffn_up.weight", "offset": 11601362944, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.13.ffn_norm.weight", "offset": 11800059904, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 11800092672, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 11800096768, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.13.attn_output.weight", "offset": 11806978048, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.13.attn_q.bias", "offset": 11862028288, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 11862061056, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.13.attn_v.bias", "offset": 11917111296, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 11917115392, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.14.attn_k.bias", "offset": 11923996672, "shape": [ 1024 ], "size": 6881280, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 11924000768, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.14.attn_output.weight", "offset": 11930882048, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.14.attn_q.bias", "offset": 11985932288, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 11985965056, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.14.attn_v.bias", "offset": 12041015296, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 12041019392, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 12047900672, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 12047933440, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.14.ffn_gate.weight", "offset": 12305293312, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.14.ffn_up.weight", "offset": 12503990272, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.14.ffn_norm.weight", "offset": 12702687232, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 12702720000, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 12702752768, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.15.ffn_gate.weight", "offset": 12960112640, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.15.ffn_up.weight", "offset": 13158809600, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.15.ffn_norm.weight", "offset": 13357506560, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 13357539328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 13357543424, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.15.attn_output.weight", "offset": 13364424704, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.15.attn_q.bias", "offset": 13419474944, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 13419507712, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.15.attn_v.bias", "offset": 13474557952, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 13474562048, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 13481443328, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.16.attn_k.bias", "offset": 13680140288, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 13680144384, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.16.attn_output.weight", "offset": 13687025664, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.16.attn_q.bias", "offset": 13742075904, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 13742108672, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.16.attn_v.bias", "offset": 13797158912, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 13797163008, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 13804044288, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 13804077056, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.16.ffn_up.weight", "offset": 14061436928, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.16.ffn_norm.weight", "offset": 14260133888, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 14260166656, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 14260199424, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.17.ffn_gate.weight", "offset": 14517559296, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.17.ffn_up.weight", "offset": 14716256256, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.17.ffn_norm.weight", "offset": 14914953216, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 14914985984, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 14914990080, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.17.attn_output.weight", "offset": 14921871360, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.17.attn_q.bias", "offset": 14976921600, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 14976954368, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.17.attn_v.bias", "offset": 15032004608, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 15032008704, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.18.ffn_gate.weight", "offset": 15038889984, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.18.ffn_up.weight", "offset": 15237586944, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.18.attn_k.bias", "offset": 15436283904, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 15436288000, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.18.attn_output.weight", "offset": 15443169280, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.18.attn_q.bias", "offset": 15498219520, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 15498252288, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.18.attn_v.bias", "offset": 15553302528, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 15553306624, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 15560187904, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 15560220672, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 15817580544, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 15817613312, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 15817646080, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.19.ffn_gate.weight", "offset": 16075005952, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 16273702912, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.19.ffn_norm.weight", "offset": 16472399872, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 16472432640, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 16472436736, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.19.attn_output.weight", "offset": 16479318016, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.19.attn_q.bias", "offset": 16534368256, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 16534401024, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.19.attn_v.bias", "offset": 16589451264, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 16589455360, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.20.attn_norm.weight", "offset": 16596336640, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 16596369408, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.20.ffn_gate.weight", "offset": 16853729280, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.20.ffn_up.weight", "offset": 17052426240, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.20.ffn_norm.weight", "offset": 17251123200, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 17251155968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 17251160064, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.20.attn_output.weight", "offset": 17258041344, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.20.attn_q.bias", "offset": 17313091584, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 17313124352, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.20.attn_v.bias", "offset": 17368174592, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 17368178688, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 17375059968, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 17375092736, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.21.ffn_gate.weight", "offset": 17632452608, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.21.ffn_up.weight", "offset": 17831149568, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.21.ffn_norm.weight", "offset": 18029846528, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 18029879296, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 18029883392, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.21.attn_output.weight", "offset": 18036764672, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.21.attn_q.bias", "offset": 18091814912, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 18091847680, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.21.attn_v.bias", "offset": 18146897920, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 18146902016, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 18153783296, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 18153816064, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.22.ffn_gate.weight", "offset": 18411175936, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.22.ffn_up.weight", "offset": 18609872896, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.22.ffn_norm.weight", "offset": 18808569856, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 18808602624, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 18808606720, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.22.attn_output.weight", "offset": 18815488000, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.22.attn_q.bias", "offset": 18870538240, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 18870571008, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.22.attn_v.bias", "offset": 18925621248, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 18925625344, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.23.attn_k.bias", "offset": 18932506624, "shape": [ 1024 ], "size": 6881280, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 18932510720, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.23.attn_output.weight", "offset": 18939392000, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.23.attn_q.bias", "offset": 18994442240, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 18994475008, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.23.attn_v.bias", "offset": 19049525248, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 19049529344, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.23.attn_norm.weight", "offset": 19056410624, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 19056443392, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.23.ffn_gate.weight", "offset": 19313803264, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.23.ffn_up.weight", "offset": 19512500224, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.23.ffn_norm.weight", "offset": 19711197184, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 19711229952, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 19711262720, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.24.ffn_gate.weight", "offset": 19968622592, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.24.ffn_up.weight", "offset": 20167319552, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.24.ffn_norm.weight", "offset": 20366016512, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 20366049280, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 20366053376, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.24.attn_output.weight", "offset": 20372934656, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.24.attn_q.bias", "offset": 20427984896, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 20428017664, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.24.attn_v.bias", "offset": 20483067904, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 20483072000, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 20489953280, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.25.attn_k.bias", "offset": 20688650240, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 20688654336, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.25.attn_output.weight", "offset": 20695535616, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.25.attn_q.bias", "offset": 20750585856, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 20750618624, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.25.attn_v.bias", "offset": 20805668864, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 20805672960, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 20812554240, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 20812587008, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.25.ffn_up.weight", "offset": 21069946880, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.25.ffn_norm.weight", "offset": 21268643840, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 21268676608, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 21268709376, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.26.ffn_gate.weight", "offset": 21526069248, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.26.ffn_up.weight", "offset": 21724766208, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.26.ffn_norm.weight", "offset": 21923463168, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 21923495936, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 21923500032, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.26.attn_output.weight", "offset": 21930381312, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.26.attn_q.bias", "offset": 21985431552, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 21985464320, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.26.attn_v.bias", "offset": 22040514560, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 22040518656, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 22047399936, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.27.ffn_up.weight", "offset": 22246096896, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.27.attn_k.bias", "offset": 22444793856, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 22444797952, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.27.attn_output.weight", "offset": 22451679232, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.27.attn_q.bias", "offset": 22506729472, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 22506762240, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.27.attn_v.bias", "offset": 22561812480, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 22561816576, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 22568697856, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 22568730624, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.27.ffn_norm.weight", "offset": 22826090496, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 22826123264, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 22826156032, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.28.ffn_gate.weight", "offset": 23083515904, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.28.ffn_up.weight", "offset": 23282212864, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.28.ffn_norm.weight", "offset": 23480909824, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 23480942592, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 23480946688, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.28.attn_output.weight", "offset": 23487827968, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.28.attn_q.bias", "offset": 23542878208, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 23542910976, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.28.attn_v.bias", "offset": 23597961216, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 23597965312, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.29.attn_norm.weight", "offset": 23604846592, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 23604879360, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.29.ffn_gate.weight", "offset": 23862239232, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.29.ffn_up.weight", "offset": 24060936192, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.29.ffn_norm.weight", "offset": 24259633152, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 24259665920, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 24259670016, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.29.attn_output.weight", "offset": 24266551296, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.29.attn_q.bias", "offset": 24321601536, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 24321634304, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.29.attn_v.bias", "offset": 24376684544, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 24376688640, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.30.attn_norm.weight", "offset": 24383569920, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 24383602688, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.30.ffn_gate.weight", "offset": 24640962560, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.30.ffn_up.weight", "offset": 24839659520, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.30.ffn_norm.weight", "offset": 25038356480, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 25038389248, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 25038393344, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.30.attn_output.weight", "offset": 25045274624, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.30.attn_q.bias", "offset": 25100324864, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 25100357632, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.30.attn_v.bias", "offset": 25155407872, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 25155411968, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.31.attn_norm.weight", "offset": 25162293248, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 25162326016, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.31.ffn_gate.weight", "offset": 25419685888, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.31.ffn_up.weight", "offset": 25618382848, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.31.ffn_norm.weight", "offset": 25817079808, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 25817112576, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 25817116672, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.31.attn_output.weight", "offset": 25823997952, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.31.attn_q.bias", "offset": 25879048192, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 25879080960, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.31.attn_v.bias", "offset": 25934131200, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 25934135296, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.32.attn_k.bias", "offset": 25941016576, "shape": [ 1024 ], "size": 6881280, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 25941020672, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.32.attn_output.weight", "offset": 25947901952, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.32.attn_q.bias", "offset": 26002952192, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 26002984960, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.32.attn_v.bias", "offset": 26058035200, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 26058039296, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.32.attn_norm.weight", "offset": 26064920576, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 26064953344, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.32.ffn_gate.weight", "offset": 26322313216, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.32.ffn_up.weight", "offset": 26521010176, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.32.ffn_norm.weight", "offset": 26719707136, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 26719739904, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 26719772672, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.33.ffn_gate.weight", "offset": 26977132544, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.33.ffn_up.weight", "offset": 27175829504, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.33.ffn_norm.weight", "offset": 27374526464, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 27374559232, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 27374563328, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.33.attn_output.weight", "offset": 27381444608, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.33.attn_q.bias", "offset": 27436494848, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 27436527616, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.33.attn_v.bias", "offset": 27491577856, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 27491581952, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.34.ffn_gate.weight", "offset": 27498463232, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.34.attn_k.bias", "offset": 27697160192, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 27697164288, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.34.attn_output.weight", "offset": 27704045568, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.34.attn_q.bias", "offset": 27759095808, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 27759128576, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.34.attn_v.bias", "offset": 27814178816, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 27814182912, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.34.attn_norm.weight", "offset": 27821064192, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 27821096960, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.34.ffn_up.weight", "offset": 28078456832, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.34.ffn_norm.weight", "offset": 28277153792, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 28277186560, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 28277219328, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.35.ffn_gate.weight", "offset": 28534579200, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.35.ffn_up.weight", "offset": 28733276160, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.35.ffn_norm.weight", "offset": 28931973120, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 28932005888, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 28932009984, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.35.attn_output.weight", "offset": 28938891264, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.35.attn_q.bias", "offset": 28993941504, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 28993974272, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.35.attn_v.bias", "offset": 29049024512, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 29049028608, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.36.ffn_gate.weight", "offset": 29055909888, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.36.ffn_up.weight", "offset": 29254606848, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.36.attn_k.bias", "offset": 29453303808, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 29453307904, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.36.attn_output.weight", "offset": 29460189184, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.36.attn_q.bias", "offset": 29515239424, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 29515272192, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.36.attn_v.bias", "offset": 29570322432, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 29570326528, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.36.attn_norm.weight", "offset": 29577207808, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 29577240576, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.36.ffn_norm.weight", "offset": 29834600448, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 29834633216, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 29834665984, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.37.ffn_gate.weight", "offset": 30092025856, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.37.ffn_up.weight", "offset": 30290722816, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.37.ffn_norm.weight", "offset": 30489419776, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 30489452544, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 30489456640, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.37.attn_output.weight", "offset": 30496337920, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.37.attn_q.bias", "offset": 30551388160, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 30551420928, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.37.attn_v.bias", "offset": 30606471168, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 30606475264, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 30613356544, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 30613389312, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.38.ffn_gate.weight", "offset": 30870749184, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.38.ffn_up.weight", "offset": 31069446144, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.38.ffn_norm.weight", "offset": 31268143104, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 31268175872, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 31268179968, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.38.attn_output.weight", "offset": 31275061248, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.38.attn_q.bias", "offset": 31330111488, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 31330144256, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.38.attn_v.bias", "offset": 31385194496, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 31385198592, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.39.attn_norm.weight", "offset": 31392079872, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 31392112640, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.39.ffn_gate.weight", "offset": 31649472512, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.39.ffn_up.weight", "offset": 31848169472, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.39.ffn_norm.weight", "offset": 32046866432, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 32046899200, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 32046903296, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.39.attn_output.weight", "offset": 32053784576, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.39.attn_q.bias", "offset": 32108834816, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 32108867584, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.39.attn_v.bias", "offset": 32163917824, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 32163921920, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.40.attn_norm.weight", "offset": 32170803200, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 32170835968, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.40.ffn_gate.weight", "offset": 32428195840, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.40.ffn_up.weight", "offset": 32626892800, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.40.ffn_norm.weight", "offset": 32825589760, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 32825622528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 32825626624, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.40.attn_output.weight", "offset": 32832507904, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.40.attn_q.bias", "offset": 32887558144, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 32887590912, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.40.attn_v.bias", "offset": 32942641152, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 32942645248, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.41.attn_k.bias", "offset": 32949526528, "shape": [ 1024 ], "size": 6881280, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 32949530624, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.41.attn_output.weight", "offset": 32956411904, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.41.attn_q.bias", "offset": 33011462144, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 33011494912, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.41.attn_v.bias", "offset": 33066545152, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 33066549248, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 33073430528, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 33073463296, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.41.ffn_gate.weight", "offset": 33330823168, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.41.ffn_up.weight", "offset": 33529520128, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.41.ffn_norm.weight", "offset": 33728217088, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 33728249856, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 33728282624, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.42.ffn_gate.weight", "offset": 33985642496, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.42.ffn_up.weight", "offset": 34184339456, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.42.ffn_norm.weight", "offset": 34383036416, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 34383069184, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 34383073280, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.42.attn_output.weight", "offset": 34389954560, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.42.attn_q.bias", "offset": 34445004800, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 34445037568, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.42.attn_v.bias", "offset": 34500087808, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 34500091904, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.43.ffn_gate.weight", "offset": 34506973184, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.43.attn_k.bias", "offset": 34705670144, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 34705674240, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.43.attn_output.weight", "offset": 34712555520, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.43.attn_q.bias", "offset": 34767605760, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 34767638528, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.43.attn_v.bias", "offset": 34822688768, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 34822692864, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.43.attn_norm.weight", "offset": 34829574144, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 34829606912, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.43.ffn_up.weight", "offset": 35086966784, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.43.ffn_norm.weight", "offset": 35285663744, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 35285696512, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 35285729280, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.44.ffn_gate.weight", "offset": 35543089152, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.44.ffn_up.weight", "offset": 35741786112, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.44.ffn_norm.weight", "offset": 35940483072, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 35940515840, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 35940519936, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.44.attn_output.weight", "offset": 35947401216, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.44.attn_q.bias", "offset": 36002451456, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 36002484224, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.44.attn_v.bias", "offset": 36057534464, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 36057538560, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.45.ffn_gate.weight", "offset": 36064419840, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.45.ffn_up.weight", "offset": 36263116800, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.45.attn_k.bias", "offset": 36461813760, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 36461817856, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.45.attn_output.weight", "offset": 36468699136, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.45.attn_q.bias", "offset": 36523749376, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 36523782144, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.45.attn_v.bias", "offset": 36578832384, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 36578836480, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.45.attn_norm.weight", "offset": 36585717760, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 36585750528, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.45.ffn_norm.weight", "offset": 36843110400, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 36843143168, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 36843175936, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.46.ffn_gate.weight", "offset": 37100535808, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.46.ffn_up.weight", "offset": 37299232768, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.46.ffn_norm.weight", "offset": 37497929728, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 37497962496, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 37497966592, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.46.attn_output.weight", "offset": 37504847872, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.46.attn_q.bias", "offset": 37559898112, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 37559930880, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.46.attn_v.bias", "offset": 37614981120, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 37614985216, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.47.attn_norm.weight", "offset": 37621866496, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 37621899264, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.47.ffn_gate.weight", "offset": 37879259136, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.47.ffn_up.weight", "offset": 38077956096, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.47.ffn_norm.weight", "offset": 38276653056, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 38276685824, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 38276689920, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.47.attn_output.weight", "offset": 38283571200, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.47.attn_q.bias", "offset": 38338621440, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 38338654208, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.47.attn_v.bias", "offset": 38393704448, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 38393708544, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.48.attn_norm.weight", "offset": 38400589824, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 38400622592, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.48.ffn_gate.weight", "offset": 38657982464, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.48.ffn_up.weight", "offset": 38856679424, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.48.ffn_norm.weight", "offset": 39055376384, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 39055409152, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 39055413248, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.48.attn_output.weight", "offset": 39062294528, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.48.attn_q.bias", "offset": 39117344768, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 39117377536, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.48.attn_v.bias", "offset": 39172427776, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 39172431872, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.49.attn_norm.weight", "offset": 39179313152, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 39179345920, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.49.ffn_gate.weight", "offset": 39436705792, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.49.ffn_up.weight", "offset": 39635402752, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.49.ffn_norm.weight", "offset": 39834099712, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 39834132480, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 39834136576, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.49.attn_output.weight", "offset": 39841017856, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.49.attn_q.bias", "offset": 39896068096, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 39896100864, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.49.attn_v.bias", "offset": 39951151104, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 39951155200, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.50.attn_k.bias", "offset": 39958036480, "shape": [ 1024 ], "size": 6881280, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 39958040576, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.50.attn_output.weight", "offset": 39964921856, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.50.attn_q.bias", "offset": 40019972096, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 40020004864, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.50.attn_v.bias", "offset": 40075055104, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 40075059200, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.50.attn_norm.weight", "offset": 40081940480, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 40081973248, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.50.ffn_gate.weight", "offset": 40339333120, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.50.ffn_up.weight", "offset": 40538030080, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.50.ffn_norm.weight", "offset": 40736727040, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 40736759808, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 40736792576, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.51.ffn_gate.weight", "offset": 40994152448, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.51.ffn_up.weight", "offset": 41192849408, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.51.ffn_norm.weight", "offset": 41391546368, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 41391579136, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 41391583232, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.51.attn_output.weight", "offset": 41398464512, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.51.attn_q.bias", "offset": 41453514752, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 41453547520, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.51.attn_v.bias", "offset": 41508597760, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 41508601856, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.52.ffn_gate.weight", "offset": 41515483136, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.52.attn_k.bias", "offset": 41714180096, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 41714184192, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.52.attn_output.weight", "offset": 41721065472, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.52.attn_q.bias", "offset": 41776115712, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 41776148480, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.52.attn_v.bias", "offset": 41831198720, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 41831202816, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.52.attn_norm.weight", "offset": 41838084096, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 41838116864, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.52.ffn_up.weight", "offset": 42095476736, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.52.ffn_norm.weight", "offset": 42294173696, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 42294206464, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 42294239232, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.53.ffn_gate.weight", "offset": 42551599104, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.53.ffn_up.weight", "offset": 42750296064, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.53.ffn_norm.weight", "offset": 42948993024, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 42949025792, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 42949029888, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.53.attn_output.weight", "offset": 42955911168, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.53.attn_q.bias", "offset": 43010961408, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 43010994176, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.53.attn_v.bias", "offset": 43066044416, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 43066048512, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.54.ffn_gate.weight", "offset": 43072929792, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.54.ffn_up.weight", "offset": 43271626752, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.54.attn_k.bias", "offset": 43470323712, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 43470327808, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.54.attn_output.weight", "offset": 43477209088, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.54.attn_q.bias", "offset": 43532259328, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 43532292096, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.54.attn_v.bias", "offset": 43587342336, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 43587346432, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.54.attn_norm.weight", "offset": 43594227712, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 43594260480, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.54.ffn_norm.weight", "offset": 43851620352, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 43851653120, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 43851685888, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.55.ffn_gate.weight", "offset": 44109045760, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.55.ffn_up.weight", "offset": 44307742720, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.55.ffn_norm.weight", "offset": 44506439680, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 44506472448, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 44506476544, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.55.attn_output.weight", "offset": 44513357824, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.55.attn_q.bias", "offset": 44568408064, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 44568440832, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.55.attn_v.bias", "offset": 44623491072, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 44623495168, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.56.attn_norm.weight", "offset": 44630376448, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 44630409216, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.56.ffn_gate.weight", "offset": 44887769088, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.56.ffn_up.weight", "offset": 45086466048, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.56.ffn_norm.weight", "offset": 45285163008, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 45285195776, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 45285199872, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.56.attn_output.weight", "offset": 45292081152, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.56.attn_q.bias", "offset": 45347131392, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 45347164160, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.56.attn_v.bias", "offset": 45402214400, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 45402218496, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.57.attn_norm.weight", "offset": 45409099776, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 45409132544, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.57.ffn_gate.weight", "offset": 45666492416, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.57.ffn_up.weight", "offset": 45865189376, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.57.ffn_norm.weight", "offset": 46063886336, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 46063919104, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 46063923200, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.57.attn_output.weight", "offset": 46070804480, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.57.attn_q.bias", "offset": 46125854720, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 46125887488, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.57.attn_v.bias", "offset": 46180937728, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 46180941824, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.58.attn_norm.weight", "offset": 46187823104, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 46187855872, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.58.ffn_gate.weight", "offset": 46445215744, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.58.ffn_up.weight", "offset": 46643912704, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.58.ffn_norm.weight", "offset": 46842609664, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 46842642432, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 46842646528, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.58.attn_output.weight", "offset": 46849527808, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.58.attn_q.bias", "offset": 46904578048, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 46904610816, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.58.attn_v.bias", "offset": 46959661056, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 46959665152, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.59.attn_k.bias", "offset": 46966546432, "shape": [ 1024 ], "size": 6881280, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 46966550528, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.59.attn_output.weight", "offset": 46973431808, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.59.attn_q.bias", "offset": 47028482048, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 47028514816, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.59.attn_v.bias", "offset": 47083565056, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 47083569152, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.59.attn_norm.weight", "offset": 47090450432, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 47090483200, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.59.ffn_gate.weight", "offset": 47347843072, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.59.ffn_up.weight", "offset": 47546540032, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.59.ffn_norm.weight", "offset": 47745236992, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 47745269760, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 47745302528, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.60.ffn_gate.weight", "offset": 48002662400, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.60.ffn_up.weight", "offset": 48201359360, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.60.ffn_norm.weight", "offset": 48400056320, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 48400089088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 48400093184, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.60.attn_output.weight", "offset": 48406974464, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.60.attn_q.bias", "offset": 48462024704, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 48462057472, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.60.attn_v.bias", "offset": 48517107712, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 48517111808, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.61.ffn_gate.weight", "offset": 48523993088, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.61.attn_k.bias", "offset": 48722690048, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 48722694144, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.61.attn_output.weight", "offset": 48729575424, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.61.attn_q.bias", "offset": 48784625664, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 48784658432, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.61.attn_v.bias", "offset": 48839708672, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 48839712768, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.61.attn_norm.weight", "offset": 48846594048, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 48846626816, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.61.ffn_up.weight", "offset": 49103986688, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.61.ffn_norm.weight", "offset": 49302683648, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 49302716416, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 49302749184, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.62.ffn_gate.weight", "offset": 49560109056, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.62.ffn_up.weight", "offset": 49758806016, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.62.ffn_norm.weight", "offset": 49957502976, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 49957535744, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 49957539840, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.62.attn_output.weight", "offset": 49964421120, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.62.attn_q.bias", "offset": 50019471360, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 50019504128, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.62.attn_v.bias", "offset": 50074554368, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 50074558464, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.63.ffn_gate.weight", "offset": 50081439744, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.63.ffn_up.weight", "offset": 50280136704, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.63.attn_k.bias", "offset": 50478833664, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 50478837760, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.63.attn_output.weight", "offset": 50485719040, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.63.attn_q.bias", "offset": 50540769280, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 50540802048, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.63.attn_v.bias", "offset": 50595852288, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 50595856384, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.63.attn_norm.weight", "offset": 50602737664, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 50602770432, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.63.ffn_norm.weight", "offset": 50860130304, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 50860163072, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 50860195840, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.64.ffn_gate.weight", "offset": 51117555712, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.64.ffn_up.weight", "offset": 51316252672, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.64.ffn_norm.weight", "offset": 51514949632, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 51514982400, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 51514986496, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.64.attn_output.weight", "offset": 51521867776, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.64.attn_q.bias", "offset": 51576918016, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 51576950784, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.64.attn_v.bias", "offset": 51632001024, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 51632005120, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.65.attn_norm.weight", "offset": 51638886400, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 51638919168, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.65.ffn_gate.weight", "offset": 51896279040, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.65.ffn_up.weight", "offset": 52094976000, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.65.ffn_norm.weight", "offset": 52293672960, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 52293705728, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 52293709824, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.65.attn_output.weight", "offset": 52300591104, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.65.attn_q.bias", "offset": 52355641344, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 52355674112, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.65.attn_v.bias", "offset": 52410724352, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 52410728448, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.66.attn_norm.weight", "offset": 52417609728, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 52417642496, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.66.ffn_gate.weight", "offset": 52675002368, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.66.ffn_up.weight", "offset": 52873699328, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.66.ffn_norm.weight", "offset": 53072396288, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 53072429056, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 53072433152, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.66.attn_output.weight", "offset": 53079314432, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.66.attn_q.bias", "offset": 53134364672, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 53134397440, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.66.attn_v.bias", "offset": 53189447680, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 53189451776, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.67.attn_norm.weight", "offset": 53196333056, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 53196365824, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.67.ffn_gate.weight", "offset": 53453725696, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.67.ffn_up.weight", "offset": 53652422656, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.67.ffn_norm.weight", "offset": 53851119616, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 53851152384, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 53851156480, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.67.attn_output.weight", "offset": 53858037760, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.67.attn_q.bias", "offset": 53913088000, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 53913120768, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.67.attn_v.bias", "offset": 53968171008, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 53968175104, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.68.attn_k.bias", "offset": 53975056384, "shape": [ 1024 ], "size": 6881280, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 53975060480, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.68.attn_output.weight", "offset": 53981941760, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.68.attn_q.bias", "offset": 54036992000, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 54037024768, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.68.attn_v.bias", "offset": 54092075008, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 54092079104, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.68.attn_norm.weight", "offset": 54098960384, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 54098993152, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.68.ffn_gate.weight", "offset": 54356353024, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.68.ffn_up.weight", "offset": 54555049984, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.68.ffn_norm.weight", "offset": 54753746944, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 54753779712, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 54753812480, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.69.ffn_gate.weight", "offset": 55011172352, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.69.ffn_up.weight", "offset": 55209869312, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.69.ffn_norm.weight", "offset": 55408566272, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 55408599040, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 55408603136, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.69.attn_output.weight", "offset": 55415484416, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.69.attn_q.bias", "offset": 55470534656, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 55470567424, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.69.attn_v.bias", "offset": 55525617664, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 55525621760, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.70.ffn_gate.weight", "offset": 55532503040, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.70.attn_k.bias", "offset": 55731200000, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 55731204096, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.70.attn_output.weight", "offset": 55738085376, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.70.attn_q.bias", "offset": 55793135616, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 55793168384, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.70.attn_v.bias", "offset": 55848218624, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 55848222720, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.70.attn_norm.weight", "offset": 55855104000, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 55855136768, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.70.ffn_up.weight", "offset": 56112496640, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.70.ffn_norm.weight", "offset": 56311193600, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 56311226368, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 56311259136, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.71.ffn_gate.weight", "offset": 56568619008, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.71.ffn_up.weight", "offset": 56767315968, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.71.ffn_norm.weight", "offset": 56966012928, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 56966045696, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 56966049792, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.71.attn_output.weight", "offset": 56972931072, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.71.attn_q.bias", "offset": 57027981312, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 57028014080, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.71.attn_v.bias", "offset": 57083064320, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 57083068416, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.72.ffn_gate.weight", "offset": 57089949696, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.72.ffn_up.weight", "offset": 57288646656, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.72.attn_k.bias", "offset": 57487343616, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 57487347712, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.72.attn_output.weight", "offset": 57494228992, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.72.attn_q.bias", "offset": 57549279232, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 57549312000, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.72.attn_v.bias", "offset": 57604362240, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 57604366336, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.72.attn_norm.weight", "offset": 57611247616, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 57611280384, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.72.ffn_norm.weight", "offset": 57868640256, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 57868673024, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 57868705792, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.73.ffn_gate.weight", "offset": 58126065664, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.73.ffn_up.weight", "offset": 58324762624, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.73.ffn_norm.weight", "offset": 58523459584, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 58523492352, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 58523496448, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.73.attn_output.weight", "offset": 58530377728, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.73.attn_q.bias", "offset": 58585427968, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 58585460736, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.73.attn_v.bias", "offset": 58640510976, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 58640515072, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.74.attn_norm.weight", "offset": 58647396352, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 58647429120, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.74.ffn_gate.weight", "offset": 58904788992, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.74.ffn_up.weight", "offset": 59103485952, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.74.ffn_norm.weight", "offset": 59302182912, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 59302215680, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 59302219776, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.74.attn_output.weight", "offset": 59309101056, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.74.attn_q.bias", "offset": 59364151296, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 59364184064, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.74.attn_v.bias", "offset": 59419234304, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 59419238400, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.75.attn_norm.weight", "offset": 59426119680, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 59426152448, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.75.ffn_gate.weight", "offset": 59683512320, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.75.ffn_up.weight", "offset": 59882209280, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.75.ffn_norm.weight", "offset": 60080906240, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 60080939008, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 60080943104, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.75.attn_output.weight", "offset": 60087824384, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.75.attn_q.bias", "offset": 60142874624, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 60142907392, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.75.attn_v.bias", "offset": 60197957632, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 60197961728, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.76.attn_norm.weight", "offset": 60204843008, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 60204875776, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.76.ffn_gate.weight", "offset": 60462235648, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.76.ffn_up.weight", "offset": 60660932608, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.76.ffn_norm.weight", "offset": 60859629568, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 60859662336, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 60859666432, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.76.attn_output.weight", "offset": 60866547712, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.76.attn_q.bias", "offset": 60921597952, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 60921630720, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.76.attn_v.bias", "offset": 60976680960, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 60976685056, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.77.attn_k.bias", "offset": 60983566336, "shape": [ 1024 ], "size": 6881280, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 60983570432, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.77.attn_output.weight", "offset": 60990451712, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.77.attn_q.bias", "offset": 61045501952, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 61045534720, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.77.attn_v.bias", "offset": 61100584960, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 61100589056, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.77.attn_norm.weight", "offset": 61107470336, "shape": [ 8192 ], "size": 6881280, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 61107503104, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.77.ffn_gate.weight", "offset": 61364862976, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.77.ffn_up.weight", "offset": 61563559936, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.77.ffn_norm.weight", "offset": 61762256896, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 61762289664, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 61762322432, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.78.ffn_gate.weight", "offset": 62019682304, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.78.ffn_up.weight", "offset": 62218379264, "shape": [ 8192, 29568 ], "size": 198696960, "type": 14 }, { "name": "blk.78.ffn_norm.weight", "offset": 62417076224, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 62417108992, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 62417113088, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.78.attn_output.weight", "offset": 62423994368, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.78.attn_q.bias", "offset": 62479044608, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 62479077376, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.78.attn_v.bias", "offset": 62534127616, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 62534131712, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.79.ffn_gate.weight", "offset": 62541012992, "shape": [ 8192, 29568 ], "size": 6881280, "type": 14 }, { "name": "blk.79.attn_k.bias", "offset": 62739709952, "shape": [ 1024 ], "size": 198696960, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 62739714048, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "blk.79.attn_output.weight", "offset": 62746595328, "shape": [ 8192, 8192 ], "size": 6881280, "type": 14 }, { "name": "blk.79.attn_q.bias", "offset": 62801645568, "shape": [ 8192 ], "size": 55050240, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 62801678336, "shape": [ 8192, 8192 ], "size": 32768, "type": 14 }, { "name": "blk.79.attn_v.bias", "offset": 62856728576, "shape": [ 1024 ], "size": 55050240, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 62856732672, "shape": [ 8192, 1024 ], "size": 4096, "type": 14 }, { "name": "output.weight", "offset": 62863613952, "shape": [ 8192, 152064 ], "size": 6881280, "type": 14 }, { "name": "blk.79.attn_norm.weight", "offset": 63885484032, "shape": [ 8192 ], "size": 1021870080, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 63885516800, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.79.ffn_up.weight", "offset": 64142876672, "shape": [ 8192, 29568 ], "size": 257359872, "type": 14 }, { "name": "blk.79.ffn_norm.weight", "offset": 64341573632, "shape": [ 8192 ], "size": 198696960, "type": 0 }, { "name": "output_norm.weight", "offset": 64341606400, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }