qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 14, "general.name": "Qwen2-72B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 80, "qwen2.context_length": 32768, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 700710912, "shape": [ 8192, 29568 ], "size": 700710912, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 836960256, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.0.attn_k.bias", "offset": 973209600, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 973213696, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 977932288, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.0.attn_q.bias", "offset": 1015681024, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 1015713792, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.0.attn_v.bias", "offset": 1053462528, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 1053466624, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 1059233792, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 1059266560, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 1240932352, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1240965120, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1240997888, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.1.ffn_gate.weight", "offset": 1422663680, "shape": [ 8192, 29568 ], "size": 181665792, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 1558913024, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 1695162368, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 1695195136, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1695199232, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 1699917824, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.1.attn_q.bias", "offset": 1737666560, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 1737699328, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.1.attn_v.bias", "offset": 1775448064, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 1775452160, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 1781219328, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1781252096, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.2.ffn_gate.weight", "offset": 1962917888, "shape": [ 8192, 29568 ], "size": 181665792, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 2099167232, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 2235416576, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 2235449344, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 2235453440, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 2240172032, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.2.attn_q.bias", "offset": 2277920768, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 2277953536, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.2.attn_v.bias", "offset": 2315702272, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 2315706368, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 2321473536, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 2321506304, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.3.ffn_gate.weight", "offset": 2503172096, "shape": [ 8192, 29568 ], "size": 181665792, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 2639421440, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 2775670784, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 2775703552, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 2775707648, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 2780426240, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.3.attn_q.bias", "offset": 2818174976, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 2818207744, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.3.attn_v.bias", "offset": 2855956480, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 2855960576, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 2861727744, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 2861760512, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.4.ffn_gate.weight", "offset": 3043426304, "shape": [ 8192, 29568 ], "size": 181665792, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 3179675648, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 3315924992, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 3315957760, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 3315961856, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 3320680448, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.4.attn_q.bias", "offset": 3358429184, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 3358461952, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.4.attn_v.bias", "offset": 3396210688, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 3396214784, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_k.bias", "offset": 3401981952, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 3401986048, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 3406704640, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.5.attn_q.bias", "offset": 3444453376, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 3444486144, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.5.attn_v.bias", "offset": 3482234880, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 3482238976, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 3488006144, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 3488038912, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.5.ffn_gate.weight", "offset": 3669704704, "shape": [ 8192, 29568 ], "size": 181665792, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 3805954048, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 3942203392, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 3942236160, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 3942268928, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.6.ffn_gate.weight", "offset": 4123934720, "shape": [ 8192, 29568 ], "size": 181665792, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 4260184064, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 4396433408, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 4396466176, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 4396470272, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 4401188864, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.6.attn_q.bias", "offset": 4438937600, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 4438970368, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.6.attn_v.bias", "offset": 4476719104, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 4476723200, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 4482490368, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.7.attn_k.bias", "offset": 4618739712, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 4618743808, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 4623462400, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.7.attn_q.bias", "offset": 4661211136, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 4661243904, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.7.attn_v.bias", "offset": 4698992640, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 4698996736, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 4704763904, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 4704796672, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 4886462464, "shape": [ 8192, 29568 ], "size": 181665792, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 5022711808, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 5022744576, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 5022777344, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.8.ffn_gate.weight", "offset": 5204443136, "shape": [ 8192, 29568 ], "size": 181665792, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 5340692480, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 5476941824, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 5476974592, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 5476978688, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 5481697280, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.8.attn_q.bias", "offset": 5519446016, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 5519478784, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.8.attn_v.bias", "offset": 5557227520, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 5557231616, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 5562998784, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 5699248128, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.9.attn_k.bias", "offset": 5835497472, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 5835501568, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 5840220160, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.9.attn_q.bias", "offset": 5877968896, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 5878001664, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.9.attn_v.bias", "offset": 5915750400, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 5915754496, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 5921521664, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 5921554432, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.10.ffn_gate.weight", "offset": 6103220224, "shape": [ 8192, 29568 ], "size": 181665792, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 6239469568, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 6375718912, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 6375751680, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 6375755776, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 6380474368, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.10.attn_q.bias", "offset": 6418223104, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 6418255872, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.10.attn_v.bias", "offset": 6456004608, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 6456008704, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 6461775872, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 6461808640, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.11.ffn_gate.weight", "offset": 6628335616, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 6764584960, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 6900834304, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 6900867072, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 6900871168, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 6905589760, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.11.attn_q.bias", "offset": 6943338496, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 6943371264, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.11.attn_v.bias", "offset": 6981120000, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 6981124096, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 6986891264, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 6986924032, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.9.ffn_norm.weight", "offset": 7153451008, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 7153483776, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 7153516544, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.12.ffn_gate.weight", "offset": 7320043520, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 7456292864, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 7592542208, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 7592574976, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 7592579072, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 7597297664, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.12.attn_q.bias", "offset": 7635046400, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 7635079168, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.12.attn_v.bias", "offset": 7672827904, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 7672832000, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 7678599168, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 7678631936, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.13.ffn_gate.weight", "offset": 7845158912, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 7981408256, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 8117657600, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 8117690368, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 8117694464, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 8122413056, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.13.attn_q.bias", "offset": 8160161792, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 8160194560, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.13.attn_v.bias", "offset": 8197943296, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 8197947392, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_k.bias", "offset": 8203714560, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 8203718656, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 8208437248, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.14.attn_q.bias", "offset": 8246185984, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 8246218752, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.14.attn_v.bias", "offset": 8283967488, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 8283971584, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 8289738752, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 8289771520, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.14.ffn_gate.weight", "offset": 8456298496, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 8592547840, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 8728797184, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 8728829952, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 8728862720, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.15.ffn_gate.weight", "offset": 8895389696, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 9031639040, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 9167888384, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 9167921152, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 9167925248, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 9172643840, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.15.attn_q.bias", "offset": 9210392576, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 9210425344, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.15.attn_v.bias", "offset": 9248174080, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 9248178176, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 9253945344, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.16.attn_k.bias", "offset": 9390194688, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 9390198784, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 9394917376, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.16.attn_q.bias", "offset": 9432666112, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 9432698880, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.16.attn_v.bias", "offset": 9470447616, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 9470451712, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 9476218880, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 9476251648, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.16.ffn_up.weight", "offset": 9642778624, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 9779027968, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 9779060736, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 9779093504, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.17.ffn_gate.weight", "offset": 9945620480, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 10081869824, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 10218119168, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 10218151936, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 10218156032, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 10222874624, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.17.attn_q.bias", "offset": 10260623360, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 10260656128, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.17.attn_v.bias", "offset": 10298404864, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 10298408960, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 10304176128, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 10440425472, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.18.attn_k.bias", "offset": 10576674816, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 10576678912, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 10581397504, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.18.attn_q.bias", "offset": 10619146240, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 10619179008, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.18.attn_v.bias", "offset": 10656927744, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 10656931840, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 10662699008, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 10662731776, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.18.ffn_norm.weight", "offset": 10829258752, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 10829291520, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 10829324288, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.19.ffn_gate.weight", "offset": 10995851264, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 11132100608, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.19.ffn_norm.weight", "offset": 11268349952, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 11268382720, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 11268386816, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 11273105408, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.19.attn_q.bias", "offset": 11310854144, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 11310886912, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.19.attn_v.bias", "offset": 11348635648, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 11348639744, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 11354406912, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 11354439680, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.20.ffn_gate.weight", "offset": 11520966656, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 11657216000, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 11793465344, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 11793498112, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 11793502208, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 11798220800, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.20.attn_q.bias", "offset": 11835969536, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 11836002304, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.20.attn_v.bias", "offset": 11873751040, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 11873755136, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 11879522304, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 11879555072, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.21.ffn_gate.weight", "offset": 12046082048, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 12182331392, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 12318580736, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 12318613504, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 12318617600, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 12323336192, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.21.attn_q.bias", "offset": 12361084928, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 12361117696, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.21.attn_v.bias", "offset": 12398866432, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 12398870528, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 12404637696, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 12404670464, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.22.ffn_gate.weight", "offset": 12571197440, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 12707446784, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 12843696128, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 12843728896, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 12843732992, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 12848451584, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.22.attn_q.bias", "offset": 12886200320, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 12886233088, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.22.attn_v.bias", "offset": 12923981824, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 12923985920, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_k.bias", "offset": 12929753088, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 12929757184, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 12934475776, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.23.attn_q.bias", "offset": 12972224512, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 12972257280, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.23.attn_v.bias", "offset": 13010006016, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 13010010112, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 13015777280, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 13015810048, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.23.ffn_gate.weight", "offset": 13182337024, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 13318586368, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 13454835712, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 13454868480, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 13454901248, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.24.ffn_gate.weight", "offset": 13621428224, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 13757677568, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 13893926912, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 13893959680, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 13893963776, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 13898682368, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.24.attn_q.bias", "offset": 13936431104, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 13936463872, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.24.attn_v.bias", "offset": 13974212608, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 13974216704, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 13979983872, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.25.attn_k.bias", "offset": 14116233216, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 14116237312, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 14120955904, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.25.attn_q.bias", "offset": 14158704640, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 14158737408, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.25.attn_v.bias", "offset": 14196486144, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 14196490240, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 14202257408, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 14202290176, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.25.ffn_up.weight", "offset": 14368817152, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 14505066496, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 14505099264, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 14505132032, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.26.ffn_gate.weight", "offset": 14671659008, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 14807908352, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 14944157696, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 14944190464, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 14944194560, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 14948913152, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.26.attn_q.bias", "offset": 14986661888, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 14986694656, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.26.attn_v.bias", "offset": 15024443392, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 15024447488, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 15030214656, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 15166464000, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.27.attn_k.bias", "offset": 15302713344, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 15302717440, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 15307436032, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.27.attn_q.bias", "offset": 15345184768, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 15345217536, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.27.attn_v.bias", "offset": 15382966272, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 15382970368, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 15388737536, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 15388770304, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.27.ffn_norm.weight", "offset": 15555297280, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 15555330048, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 15555362816, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.28.ffn_gate.weight", "offset": 15721889792, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 15858139136, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.28.ffn_norm.weight", "offset": 15994388480, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 15994421248, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 15994425344, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 15999143936, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.28.attn_q.bias", "offset": 16036892672, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 16036925440, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.28.attn_v.bias", "offset": 16074674176, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 16074678272, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 16080445440, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 16080478208, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.29.ffn_gate.weight", "offset": 16247005184, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 16383254528, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.29.ffn_norm.weight", "offset": 16519503872, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 16519536640, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 16519540736, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 16524259328, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.29.attn_q.bias", "offset": 16562008064, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 16562040832, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.29.attn_v.bias", "offset": 16599789568, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 16599793664, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 16605560832, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 16605593600, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.30.ffn_gate.weight", "offset": 16772120576, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 16908369920, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.30.ffn_norm.weight", "offset": 17044619264, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 17044652032, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 17044656128, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 17049374720, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.30.attn_q.bias", "offset": 17087123456, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 17087156224, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.30.attn_v.bias", "offset": 17124904960, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 17124909056, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 17130676224, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 17130708992, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.31.ffn_gate.weight", "offset": 17297235968, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 17433485312, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.31.ffn_norm.weight", "offset": 17569734656, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 17569767424, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 17569771520, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.31.attn_output.weight", "offset": 17574490112, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.31.attn_q.bias", "offset": 17612238848, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 17612271616, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.31.attn_v.bias", "offset": 17650020352, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 17650024448, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_k.bias", "offset": 17655791616, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 17655795712, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.32.attn_output.weight", "offset": 17660514304, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.32.attn_q.bias", "offset": 17698263040, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 17698295808, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.32.attn_v.bias", "offset": 17736044544, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 17736048640, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 17741815808, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 17741848576, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.32.ffn_gate.weight", "offset": 17908375552, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 18044624896, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.32.ffn_norm.weight", "offset": 18180874240, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 18180907008, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 18180939776, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.33.ffn_gate.weight", "offset": 18347466752, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 18483716096, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.33.ffn_norm.weight", "offset": 18619965440, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 18619998208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 18620002304, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.33.attn_output.weight", "offset": 18624720896, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.33.attn_q.bias", "offset": 18662469632, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 18662502400, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.33.attn_v.bias", "offset": 18700251136, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 18700255232, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.ffn_gate.weight", "offset": 18706022400, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.34.attn_k.bias", "offset": 18842271744, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 18842275840, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.34.attn_output.weight", "offset": 18846994432, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.34.attn_q.bias", "offset": 18884743168, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 18884775936, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.34.attn_v.bias", "offset": 18922524672, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 18922528768, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 18928295936, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 18928328704, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.34.ffn_up.weight", "offset": 19094855680, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.34.ffn_norm.weight", "offset": 19231105024, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 19231137792, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 19231170560, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.35.ffn_gate.weight", "offset": 19397697536, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 19533946880, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.35.ffn_norm.weight", "offset": 19670196224, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 19670228992, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 19670233088, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 19674951680, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.35.attn_q.bias", "offset": 19712700416, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 19712733184, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.35.attn_v.bias", "offset": 19750481920, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 19750486016, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 19756253184, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 19892502528, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.36.attn_k.bias", "offset": 20028751872, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 20028755968, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.36.attn_output.weight", "offset": 20033474560, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.36.attn_q.bias", "offset": 20071223296, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 20071256064, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.36.attn_v.bias", "offset": 20109004800, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 20109008896, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 20114776064, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 20114808832, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.36.ffn_norm.weight", "offset": 20281335808, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 20281368576, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 20281401344, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.37.ffn_gate.weight", "offset": 20447928320, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 20584177664, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.37.ffn_norm.weight", "offset": 20720427008, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 20720459776, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 20720463872, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.37.attn_output.weight", "offset": 20725182464, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.37.attn_q.bias", "offset": 20762931200, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 20762963968, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.37.attn_v.bias", "offset": 20800712704, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 20800716800, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 20806483968, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 20806516736, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.38.ffn_gate.weight", "offset": 20973043712, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 21109293056, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.38.ffn_norm.weight", "offset": 21245542400, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 21245575168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 21245579264, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.38.attn_output.weight", "offset": 21250297856, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.38.attn_q.bias", "offset": 21288046592, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 21288079360, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.38.attn_v.bias", "offset": 21325828096, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 21325832192, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 21331599360, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 21331632128, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.39.ffn_gate.weight", "offset": 21498159104, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 21634408448, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.39.ffn_norm.weight", "offset": 21770657792, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 21770690560, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 21770694656, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.39.attn_output.weight", "offset": 21775413248, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.39.attn_q.bias", "offset": 21813161984, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 21813194752, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.39.attn_v.bias", "offset": 21850943488, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 21850947584, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 21856714752, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 21856747520, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.40.ffn_gate.weight", "offset": 22023274496, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.40.ffn_up.weight", "offset": 22159523840, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.40.ffn_norm.weight", "offset": 22295773184, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 22295805952, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 22295810048, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.40.attn_output.weight", "offset": 22300528640, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.40.attn_q.bias", "offset": 22338277376, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 22338310144, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.40.attn_v.bias", "offset": 22376058880, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 22376062976, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_k.bias", "offset": 22381830144, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 22381834240, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.41.attn_output.weight", "offset": 22386552832, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.41.attn_q.bias", "offset": 22424301568, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 22424334336, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.41.attn_v.bias", "offset": 22462083072, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 22462087168, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 22467854336, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 22467887104, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.41.ffn_gate.weight", "offset": 22634414080, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.41.ffn_up.weight", "offset": 22770663424, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.41.ffn_norm.weight", "offset": 22906912768, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 22906945536, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 22906978304, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.42.ffn_gate.weight", "offset": 23073505280, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.42.ffn_up.weight", "offset": 23209754624, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.42.ffn_norm.weight", "offset": 23346003968, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 23346036736, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 23346040832, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.42.attn_output.weight", "offset": 23350759424, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.42.attn_q.bias", "offset": 23388508160, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 23388540928, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.42.attn_v.bias", "offset": 23426289664, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 23426293760, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.ffn_gate.weight", "offset": 23432060928, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.43.attn_k.bias", "offset": 23568310272, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 23568314368, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.43.attn_output.weight", "offset": 23573032960, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.43.attn_q.bias", "offset": 23610781696, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 23610814464, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.43.attn_v.bias", "offset": 23648563200, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 23648567296, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 23654334464, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 23654367232, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.43.ffn_up.weight", "offset": 23820894208, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.43.ffn_norm.weight", "offset": 23957143552, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 23957176320, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 23957209088, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.44.ffn_gate.weight", "offset": 24123736064, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.44.ffn_up.weight", "offset": 24259985408, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.44.ffn_norm.weight", "offset": 24396234752, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 24396267520, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 24396271616, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.44.attn_output.weight", "offset": 24400990208, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.44.attn_q.bias", "offset": 24438738944, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 24438771712, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.44.attn_v.bias", "offset": 24476520448, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 24476524544, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.ffn_gate.weight", "offset": 24482291712, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.45.ffn_up.weight", "offset": 24618541056, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.45.attn_k.bias", "offset": 24754790400, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 24754794496, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.45.attn_output.weight", "offset": 24759513088, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.45.attn_q.bias", "offset": 24797261824, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 24797294592, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.45.attn_v.bias", "offset": 24835043328, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 24835047424, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 24840814592, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 24840847360, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.45.ffn_norm.weight", "offset": 25007374336, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 25007407104, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 25007439872, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.46.ffn_gate.weight", "offset": 25173966848, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.46.ffn_up.weight", "offset": 25310216192, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.46.ffn_norm.weight", "offset": 25446465536, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 25446498304, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 25446502400, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.46.attn_output.weight", "offset": 25451220992, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "\u0000\u0000\u0000blk.58.attn_out", "offset": 25488969728, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 25489002496, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.46.attn_v.bias", "offset": 25526751232, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 25526755328, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 25532522496, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 25532555264, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.47.ffn_gate.weight", "offset": 25699082240, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.47.ffn_up.weight", "offset": 25835331584, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.47.ffn_norm.weight", "offset": 25971580928, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 25971613696, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 25971617792, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.47.attn_output.weight", "offset": 25976336384, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.47.attn_q.bias", "offset": 26014085120, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 26014117888, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.47.attn_v.bias", "offset": 26051866624, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 26051870720, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.48.attn_norm.weight", "offset": 26057637888, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 26057670656, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.48.ffn_gate.weight", "offset": 26224197632, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.48.ffn_up.weight", "offset": 26360446976, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.48.ffn_norm.weight", "offset": 26496696320, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 26496729088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 26496733184, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.48.attn_output.weight", "offset": 26501451776, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.48.attn_q.bias", "offset": 26539200512, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 26539233280, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.48.attn_v.bias", "offset": 26576982016, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 26576986112, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.49.attn_norm.weight", "offset": 26582753280, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 26582786048, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.49.ffn_gate.weight", "offset": 26749313024, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.49.ffn_up.weight", "offset": 26885562368, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.49.ffn_norm.weight", "offset": 27021811712, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 27021844480, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 27021848576, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.49.attn_output.weight", "offset": 27026567168, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.49.attn_q.bias", "offset": 27064315904, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 27064348672, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.49.attn_v.bias", "offset": 27102097408, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 27102101504, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_k.bias", "offset": 27107868672, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 27107872768, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.50.attn_output.weight", "offset": 27112591360, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.50.attn_q.bias", "offset": 27150340096, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 27150372864, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.50.attn_v.bias", "offset": 27188121600, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 27188125696, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_norm.weight", "offset": 27193892864, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 27193925632, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.50.ffn_gate.weight", "offset": 27360452608, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.50.ffn_up.weight", "offset": 27496701952, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.50.ffn_norm.weight", "offset": 27632951296, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 27632984064, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 27633016832, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.51.ffn_gate.weight", "offset": 27799543808, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.51.ffn_up.weight", "offset": 27935793152, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.51.ffn_norm.weight", "offset": 28072042496, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 28072075264, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 28072079360, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.51.attn_output.weight", "offset": 28076797952, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.51.attn_q.bias", "offset": 28114546688, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 28114579456, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.51.attn_v.bias", "offset": 28152328192, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 28152332288, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.ffn_gate.weight", "offset": 28158099456, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.52.attn_k.bias", "offset": 28294348800, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 28294352896, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.52.attn_output.weight", "offset": 28299071488, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.52.attn_q.bias", "offset": 28336820224, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 28336852992, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.52.attn_v.bias", "offset": 28374601728, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 28374605824, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.attn_norm.weight", "offset": 28380372992, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 28380405760, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.52.ffn_up.weight", "offset": 28546932736, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.52.ffn_norm.weight", "offset": 28683182080, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 28683214848, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 28683247616, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.53.ffn_gate.weight", "offset": 28849774592, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.53.ffn_up.weight", "offset": 28986023936, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.53.ffn_norm.weight", "offset": 29122273280, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 29122306048, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 29122310144, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.53.attn_output.weight", "offset": 29127028736, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.53.attn_q.bias", "offset": 29164777472, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 29164810240, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.53.attn_v.bias", "offset": 29202558976, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 29202563072, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.ffn_gate.weight", "offset": 29208330240, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.54.ffn_up.weight", "offset": 29344579584, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.54.attn_k.bias", "offset": 29480828928, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 29480833024, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.54.attn_output.weight", "offset": 29485551616, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.54.attn_q.bias", "offset": 29523300352, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 29523333120, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.54.attn_v.bias", "offset": 29561081856, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 29561085952, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.attn_norm.weight", "offset": 29566853120, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 29566885888, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.54.ffn_norm.weight", "offset": 29733412864, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 29733445632, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 29733478400, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.55.ffn_gate.weight", "offset": 29900005376, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.55.ffn_up.weight", "offset": 30036254720, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.55.ffn_norm.weight", "offset": 30172504064, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 30172536832, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 30172540928, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.55.attn_output.weight", "offset": 30177259520, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.55.attn_q.bias", "offset": 30215008256, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 30215041024, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.55.attn_v.bias", "offset": 30252789760, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 30252793856, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.56.attn_norm.weight", "offset": 30258561024, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 30258593792, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.56.ffn_gate.weight", "offset": 30425120768, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.56.ffn_up.weight", "offset": 30561370112, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.56.ffn_norm.weight", "offset": 30697619456, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 30697652224, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 30697656320, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.56.attn_output.weight", "offset": 30702374912, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.56.attn_q.bias", "offset": 30740123648, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 30740156416, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.56.attn_v.bias", "offset": 30777905152, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 30777909248, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.57.attn_norm.weight", "offset": 30783676416, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 30783709184, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.57.ffn_gate.weight", "offset": 30950236160, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.57.ffn_up.weight", "offset": 31086485504, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.57.ffn_norm.weight", "offset": 31222734848, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 31222767616, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 31222771712, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.57.attn_output.weight", "offset": 31227490304, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.57.attn_q.bias", "offset": 31265239040, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 31265271808, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.57.attn_v.bias", "offset": 31303020544, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 31303024640, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.58.attn_norm.weight", "offset": 31308791808, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 31308824576, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.58.ffn_gate.weight", "offset": 31475351552, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.58.ffn_up.weight", "offset": 31611600896, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.58.ffn_norm.weight", "offset": 31747850240, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 31747883008, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 31747887104, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.58.attn_output.weight", "offset": 31752605696, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.58.attn_q.bias", "offset": 31790354432, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 31790387200, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.58.attn_v.bias", "offset": 31828135936, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 31828140032, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_k.bias", "offset": 31833907200, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 31833911296, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.59.attn_output.weight", "offset": 31838629888, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.59.attn_q.bias", "offset": 31876378624, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 31876411392, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.59.attn_v.bias", "offset": 31914160128, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 31914164224, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_norm.weight", "offset": 31919931392, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 31919964160, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.59.ffn_gate.weight", "offset": 32086491136, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.59.ffn_up.weight", "offset": 32222740480, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.59.ffn_norm.weight", "offset": 32358989824, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 32359022592, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 32359055360, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.60.ffn_gate.weight", "offset": 32525582336, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.60.ffn_up.weight", "offset": 32661831680, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.60.ffn_norm.weight", "offset": 32798081024, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 32798113792, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 32798117888, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.60.attn_output.weight", "offset": 32802836480, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.60.attn_q.bias", "offset": 32840585216, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 32840617984, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.60.attn_v.bias", "offset": 32878366720, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 32878370816, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.ffn_gate.weight", "offset": 32884137984, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.61.attn_k.bias", "offset": 33020387328, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 33020391424, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.61.attn_output.weight", "offset": 33025110016, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.61.attn_q.bias", "offset": 33062858752, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 33062891520, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.61.attn_v.bias", "offset": 33100640256, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 33100644352, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.attn_norm.weight", "offset": 33106411520, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 33106444288, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.61.ffn_up.weight", "offset": 33272971264, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.61.ffn_norm.weight", "offset": 33409220608, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 33409253376, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 33409286144, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.62.ffn_gate.weight", "offset": 33575813120, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.62.ffn_up.weight", "offset": 33712062464, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.62.ffn_norm.weight", "offset": 33848311808, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 33848344576, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 33848348672, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.62.attn_output.weight", "offset": 33853067264, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.62.attn_q.bias", "offset": 33890816000, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 33890848768, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.62.attn_v.bias", "offset": 33928597504, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 33928601600, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.ffn_gate.weight", "offset": 33934368768, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.63.ffn_up.weight", "offset": 34070618112, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.63.attn_k.bias", "offset": 34206867456, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 34206871552, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.63.attn_output.weight", "offset": 34211590144, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.63.attn_q.bias", "offset": 34249338880, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 34249371648, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.63.attn_v.bias", "offset": 34287120384, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 34287124480, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.attn_norm.weight", "offset": 34292891648, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 34292924416, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.63.ffn_norm.weight", "offset": 34459451392, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 34459484160, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 34459516928, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.64.ffn_gate.weight", "offset": 34626043904, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.64.ffn_up.weight", "offset": 34762293248, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.64.ffn_norm.weight", "offset": 34898542592, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 34898575360, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 34898579456, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.64.attn_output.weight", "offset": 34903298048, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.64.attn_q.bias", "offset": 34941046784, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 34941079552, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.64.attn_v.bias", "offset": 34978828288, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 34978832384, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.65.attn_norm.weight", "offset": 34984599552, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 34984632320, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.65.ffn_gate.weight", "offset": 35151159296, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.65.ffn_up.weight", "offset": 35287408640, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.65.ffn_norm.weight", "offset": 35423657984, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 35423690752, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 35423694848, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.65.attn_output.weight", "offset": 35428413440, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.65.attn_q.bias", "offset": 35466162176, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 35466194944, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.65.attn_v.bias", "offset": 35503943680, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 35503947776, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.66.attn_norm.weight", "offset": 35509714944, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 35509747712, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.66.ffn_gate.weight", "offset": 35676274688, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.66.ffn_up.weight", "offset": 35812524032, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.66.ffn_norm.weight", "offset": 35948773376, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 35948806144, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 35948810240, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.66.attn_output.weight", "offset": 35953528832, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.66.attn_q.bias", "offset": 35991277568, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 35991310336, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.66.attn_v.bias", "offset": 36029059072, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 36029063168, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.67.attn_norm.weight", "offset": 36034830336, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 36034863104, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.67.ffn_gate.weight", "offset": 36201390080, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.67.ffn_up.weight", "offset": 36337639424, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.67.ffn_norm.weight", "offset": 36473888768, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 36473921536, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 36473925632, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.67.attn_output.weight", "offset": 36478644224, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.67.attn_q.bias", "offset": 36516392960, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 36516425728, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.67.attn_v.bias", "offset": 36554174464, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 36554178560, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_k.bias", "offset": 36559945728, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 36559949824, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.68.attn_output.weight", "offset": 36564668416, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.68.attn_q.bias", "offset": 36602417152, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 36602449920, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.68.attn_v.bias", "offset": 36640198656, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 36640202752, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_norm.weight", "offset": 36645969920, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 36646002688, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.68.ffn_gate.weight", "offset": 36812529664, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.68.ffn_up.weight", "offset": 36948779008, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.68.ffn_norm.weight", "offset": 37085028352, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 37085061120, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 37085093888, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.69.ffn_gate.weight", "offset": 37251620864, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.69.ffn_up.weight", "offset": 37387870208, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.69.ffn_norm.weight", "offset": 37524119552, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 37524152320, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 37524156416, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.69.attn_output.weight", "offset": 37528875008, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.69.attn_q.bias", "offset": 37566623744, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 37566656512, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.69.attn_v.bias", "offset": 37604405248, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 37604409344, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.ffn_gate.weight", "offset": 37610176512, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.70.attn_k.bias", "offset": 37746425856, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 37746429952, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.70.attn_output.weight", "offset": 37751148544, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.70.attn_q.bias", "offset": 37788897280, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 37788930048, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.70.attn_v.bias", "offset": 37826678784, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 37826682880, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.attn_norm.weight", "offset": 37832450048, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 37832482816, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.70.ffn_up.weight", "offset": 37999009792, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.70.ffn_norm.weight", "offset": 38135259136, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 38135291904, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 38135324672, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.71.ffn_gate.weight", "offset": 38301851648, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.71.ffn_up.weight", "offset": 38438100992, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.71.ffn_norm.weight", "offset": 38574350336, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 38574383104, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 38574387200, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.71.attn_output.weight", "offset": 38579105792, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.71.attn_q.bias", "offset": 38616854528, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 38616887296, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.71.attn_v.bias", "offset": 38654636032, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 38654640128, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.ffn_gate.weight", "offset": 38660407296, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.72.ffn_up.weight", "offset": 38796656640, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.72.attn_k.bias", "offset": 38932905984, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 38932910080, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.72.attn_output.weight", "offset": 38937628672, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.72.attn_q.bias", "offset": 38975377408, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 38975410176, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.72.attn_v.bias", "offset": 39013158912, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 39013163008, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.attn_norm.weight", "offset": 39018930176, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 39018962944, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.72.ffn_norm.weight", "offset": 39185489920, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 39185522688, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 39185555456, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.73.ffn_gate.weight", "offset": 39352082432, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.73.ffn_up.weight", "offset": 39488331776, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.73.ffn_norm.weight", "offset": 39624581120, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 39624613888, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 39624617984, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.73.attn_output.weight", "offset": 39629336576, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.73.attn_q.bias", "offset": 39667085312, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 39667118080, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.73.attn_v.bias", "offset": 39704866816, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 39704870912, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.74.attn_norm.weight", "offset": 39710638080, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 39710670848, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.74.ffn_gate.weight", "offset": 39877197824, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.74.ffn_up.weight", "offset": 40013447168, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.74.ffn_norm.weight", "offset": 40149696512, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 40149729280, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 40149733376, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.74.attn_output.weight", "offset": 40154451968, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.74.attn_q.bias", "offset": 40192200704, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 40192233472, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.74.attn_v.bias", "offset": 40229982208, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 40229986304, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.75.attn_norm.weight", "offset": 40235753472, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 40235786240, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.75.ffn_gate.weight", "offset": 40402313216, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.75.ffn_up.weight", "offset": 40538562560, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.75.ffn_norm.weight", "offset": 40674811904, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 40674844672, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 40674848768, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.75.attn_output.weight", "offset": 40679567360, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.75.attn_q.bias", "offset": 40717316096, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 40717348864, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.75.attn_v.bias", "offset": 40755097600, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 40755101696, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.76.attn_norm.weight", "offset": 40760868864, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 40760901632, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.76.ffn_gate.weight", "offset": 40927428608, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.76.ffn_up.weight", "offset": 41063677952, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.76.ffn_norm.weight", "offset": 41199927296, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 41199960064, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 41199964160, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.76.attn_output.weight", "offset": 41204682752, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.76.attn_q.bias", "offset": 41242431488, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 41242464256, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.76.attn_v.bias", "offset": 41280212992, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 41280217088, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_k.bias", "offset": 41285984256, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 41285988352, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.77.attn_output.weight", "offset": 41290706944, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.77.attn_q.bias", "offset": 41328455680, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 41328488448, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.77.attn_v.bias", "offset": 41366237184, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 41366241280, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_norm.weight", "offset": 41372008448, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 41372041216, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.77.ffn_gate.weight", "offset": 41538568192, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.77.ffn_up.weight", "offset": 41674817536, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.77.ffn_norm.weight", "offset": 41811066880, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 41811099648, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 41811132416, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.78.ffn_gate.weight", "offset": 41977659392, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.78.ffn_up.weight", "offset": 42113908736, "shape": [ 8192, 29568 ], "size": 136249344, "type": 12 }, { "name": "blk.78.ffn_norm.weight", "offset": 42250158080, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 42250190848, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 42250194944, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.78.attn_output.weight", "offset": 42254913536, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.78.attn_q.bias", "offset": 42292662272, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 42292695040, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.78.attn_v.bias", "offset": 42330443776, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 42330447872, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.79.ffn_gate.weight", "offset": 42336215040, "shape": [ 8192, 29568 ], "size": 5767168, "type": 12 }, { "name": "blk.79.attn_k.bias", "offset": 42472464384, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 42472468480, "shape": [ 8192, 1024 ], "size": 4096, "type": 12 }, { "name": "blk.79.attn_output.weight", "offset": 42477187072, "shape": [ 8192, 8192 ], "size": 4718592, "type": 12 }, { "name": "blk.79.attn_q.bias", "offset": 42514935808, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 42514968576, "shape": [ 8192, 8192 ], "size": 32768, "type": 12 }, { "name": "blk.79.attn_v.bias", "offset": 42552717312, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 42552721408, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "output.weight", "offset": 42558488576, "shape": [ 8192, 152064 ], "size": 5767168, "type": 14 }, { "name": "blk.79.attn_norm.weight", "offset": 43580358656, "shape": [ 8192 ], "size": 1021870080, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 43580391424, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.79.ffn_up.weight", "offset": 43746918400, "shape": [ 8192, 29568 ], "size": 166526976, "type": 12 }, { "name": "blk.79.ffn_norm.weight", "offset": 43883167744, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "output_norm.weight", "offset": 43883200512, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }