qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 13, "general.name": "Qwen2-72B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 80, "qwen2.context_length": 32768, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 535265280, "shape": [ 8192, 29568 ], "size": 535265280, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 639344640, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.0.attn_k.bias", "offset": 743424000, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 743428096, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 747032576, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.0.attn_q.bias", "offset": 793169920, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 793202688, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.0.attn_v.bias", "offset": 822038528, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 822042624, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 827809792, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 827842560, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 1009508352, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1009541120, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1009573888, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.1.ffn_gate.weight", "offset": 1191239680, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 1295319040, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 1399398400, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 1399431168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1399435264, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 1403039744, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.1.attn_q.bias", "offset": 1449177088, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 1449209856, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.1.attn_v.bias", "offset": 1478045696, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 1478049792, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 1483816960, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1483849728, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.2.ffn_gate.weight", "offset": 1665515520, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 1769594880, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 1873674240, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 1873707008, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1873711104, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 1877315584, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.2.attn_q.bias", "offset": 1923452928, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 1923485696, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.2.attn_v.bias", "offset": 1952321536, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 1952325632, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 1958092800, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1958125568, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.3.ffn_gate.weight", "offset": 2139791360, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 2243870720, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 2347950080, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 2347982848, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 2347986944, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 2351591424, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.3.attn_q.bias", "offset": 2397728768, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 2397761536, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.3.attn_v.bias", "offset": 2426597376, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 2426601472, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 2432368640, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 2432401408, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.4.ffn_gate.weight", "offset": 2614067200, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 2718146560, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 2822225920, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 2822258688, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 2822262784, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 2825867264, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.4.attn_q.bias", "offset": 2872004608, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 2872037376, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.4.attn_v.bias", "offset": 2900873216, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "\u0000\u0000\u0016\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.16.ff", "offset": 2900877312, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_k.bias", "offset": 2906644480, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 2906648576, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 2910253056, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.5.attn_q.bias", "offset": 2956390400, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 2956423168, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.5.attn_v.bias", "offset": 2985259008, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 2985263104, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 2991030272, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 2991063040, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.5.ffn_gate.weight", "offset": 3172728832, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 3276808192, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 3380887552, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 3380920320, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 3380953088, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.6.ffn_gate.weight", "offset": 3562618880, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 3666698240, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 3770777600, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 3770810368, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 3770814464, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 3774418944, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.6.attn_q.bias", "offset": 3820556288, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 3820589056, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.6.attn_v.bias", "offset": 3849424896, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 3849428992, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 3855196160, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.7.attn_k.bias", "offset": 3959275520, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 3959279616, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 3962884096, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.7.attn_q.bias", "offset": 4009021440, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 4009054208, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.7.attn_v.bias", "offset": 4037890048, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 4037894144, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 4043661312, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 4043694080, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 4225359872, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 4329439232, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 4329472000, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 4329504768, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.8.ffn_gate.weight", "offset": 4511170560, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 4615249920, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 4719329280, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 4719362048, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 4719366144, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 4722970624, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.8.attn_q.bias", "offset": 4769107968, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 4769140736, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.8.attn_v.bias", "offset": 4797976576, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 4797980672, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 4803747840, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 4907827200, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.9.attn_k.bias", "offset": 5011906560, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 5011910656, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 5015515136, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.9.attn_q.bias", "offset": 5061652480, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 5061685248, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.9.attn_v.bias", "offset": 5090521088, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 5090525184, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 5096292352, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 5096325120, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.10.ffn_gate.weight", "offset": 5277990912, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 5382070272, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 5486149632, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 5486182400, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 5486186496, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 5489790976, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.10.attn_q.bias", "offset": 5535928320, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 5535961088, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.10.attn_v.bias", "offset": 5564796928, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 5564801024, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 5570568192, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 5570600960, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.11.ffn_gate.weight", "offset": 5752266752, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 5856346112, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 5960425472, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 5960458240, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 5960462336, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 5964066816, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.11.attn_q.bias", "offset": 6010204160, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 6010236928, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.11.attn_v.bias", "offset": 6039072768, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 6039076864, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 6044844032, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 6044876800, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 6226542592, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 6226575360, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 6226608128, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.12.ffn_gate.weight", "offset": 6408273920, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 6512353280, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 6616432640, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 6616465408, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 6616469504, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 6620073984, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.12.attn_q.bias", "offset": 6666211328, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 6666244096, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.12.attn_v.bias", "offset": 6695079936, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 6695084032, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 6700851200, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 6700883968, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.13.ffn_gate.weight", "offset": 6882549760, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 6986629120, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 7090708480, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 7090741248, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 7090745344, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 7094349824, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.13.attn_q.bias", "offset": 7140487168, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 7140519936, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.13.attn_v.bias", "offset": 7169355776, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 7169359872, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_k.bias", "offset": 7175127040, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 7175131136, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 7178735616, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.14.attn_q.bias", "offset": 7224872960, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 7224905728, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.14.attn_v.bias", "offset": 7253741568, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 7253745664, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 7259512832, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 7259545600, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.14.ffn_gate.weight", "offset": 7441211392, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 7545290752, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 7649370112, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 7649402880, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 7649435648, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.15.ffn_gate.weight", "offset": 7831101440, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 7935180800, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 8039260160, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 8039292928, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 8039297024, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 8042901504, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.15.attn_q.bias", "offset": 8089038848, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 8089071616, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.15.attn_v.bias", "offset": 8117907456, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 8117911552, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 8123678720, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.16.attn_k.bias", "offset": 8227758080, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 8227762176, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 8231366656, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.16.attn_q.bias", "offset": 8277504000, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 8277536768, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.16.attn_v.bias", "offset": 8306372608, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 8306376704, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 8312143872, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 8312176640, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 8493842432, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "\u0000\u0000\u0000\u0000blk.28.attn_v.bias", "offset": 8597921792, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 8597954560, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 8597987328, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.17.ffn_gate.weight", "offset": 8779653120, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 8883732480, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 8987811840, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 8987844608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 8987848704, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 8991453184, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.17.attn_q.bias", "offset": 9037590528, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 9037623296, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.17.attn_v.bias", "offset": 9066459136, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 9066463232, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 9072230400, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 9176309760, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.18.attn_k.bias", "offset": 9280389120, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 9280393216, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 9283997696, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.18.attn_q.bias", "offset": 9330135040, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 9330167808, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.18.attn_v.bias", "offset": 9359003648, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 9359007744, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 9364774912, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 9364807680, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 9546473472, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 9546506240, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 9546539008, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.19.ffn_gate.weight", "offset": 9728204800, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 9832284160, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 9936363520, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 9936396288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 9936400384, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 9940004864, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.19.attn_q.bias", "offset": 9986142208, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 9986174976, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.19.attn_v.bias", "offset": 10015010816, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 10015014912, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 10020782080, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 10020814848, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.20.ffn_gate.weight", "offset": 10202480640, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 10306560000, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 10410639360, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 10410672128, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 10410676224, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 10414280704, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.20.attn_q.bias", "offset": 10460418048, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 10460450816, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.20.attn_v.bias", "offset": 10489286656, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 10489290752, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 10495057920, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 10495090688, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.21.ffn_gate.weight", "offset": 10676756480, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 10780835840, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 10884915200, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 10884947968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 10884952064, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 10888556544, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.21.attn_q.bias", "offset": 10934693888, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 10934726656, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.21.attn_v.bias", "offset": 10963562496, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 10963566592, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 10969333760, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 10969366528, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.22.ffn_gate.weight", "offset": 11151032320, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 11255111680, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 11359191040, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 11359223808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 11359227904, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 11362832384, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.22.attn_q.bias", "offset": 11408969728, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 11409002496, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.22.attn_v.bias", "offset": 11437838336, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 11437842432, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_k.bias", "offset": 11443609600, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 11443613696, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 11447218176, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.23.attn_q.bias", "offset": 11493355520, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 11493388288, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.23.attn_v.bias", "offset": 11522224128, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 11522228224, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 11527995392, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 11528028160, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.23.ffn_gate.weight", "offset": 11709693952, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 11813773312, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 11917852672, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 11917885440, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 11917918208, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.24.ffn_gate.weight", "offset": 12099584000, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 12203663360, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 12307742720, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 12307775488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 12307779584, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 12311384064, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.24.attn_q.bias", "offset": 12357521408, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 12357554176, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.24.attn_v.bias", "offset": 12386390016, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 12386394112, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 12392161280, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.25.attn_k.bias", "offset": 12496240640, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 12496244736, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 12499849216, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.25.attn_q.bias", "offset": 12545986560, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 12546019328, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.25.attn_v.bias", "offset": 12574855168, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 12574859264, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 12580626432, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 12580659200, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.25.ffn_up.weight", "offset": 12762324992, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 12866404352, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 12866437120, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 12866469888, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.26.ffn_gate.weight", "offset": 13048135680, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 13152215040, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 13256294400, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 13256327168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 13256331264, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 13259935744, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.26.attn_q.bias", "offset": 13306073088, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 13306105856, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.26.attn_v.bias", "offset": 13334941696, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 13334945792, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 13340712960, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 13444792320, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.27.attn_k.bias", "offset": 13548871680, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 13548875776, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 13552480256, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.27.attn_q.bias", "offset": 13598617600, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 13598650368, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.27.attn_v.bias", "offset": 13627486208, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 13627490304, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 13633257472, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 13633290240, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.27.ffn_norm.weight", "offset": 13814956032, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 13814988800, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 13815021568, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.28.ffn_gate.weight", "offset": 13996687360, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 14100766720, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 14204846080, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 14204878848, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 14204882944, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 14208487424, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.28.attn_q.bias", "offset": 14254624768, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 14254657536, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "\u0000\u0000\u0000\u0000blk.40.attn_q.", "offset": 14283493376, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 14283497472, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 14289264640, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 14289297408, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.29.ffn_gate.weight", "offset": 14470963200, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 14575042560, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 14679121920, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 14679154688, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 14679158784, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 14682763264, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.29.attn_q.bias", "offset": 14728900608, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 14728933376, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.29.attn_v.bias", "offset": 14757769216, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 14757773312, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 14763540480, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 14763573248, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.30.ffn_gate.weight", "offset": 14945239040, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 15049318400, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 15153397760, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 15153430528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 15153434624, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 15157039104, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.30.attn_q.bias", "offset": 15203176448, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 15203209216, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.30.attn_v.bias", "offset": 15232045056, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 15232049152, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 15237816320, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 15237849088, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.31.ffn_gate.weight", "offset": 15419514880, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 15523594240, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 15627673600, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 15627706368, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 15627710464, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.31.attn_output.weight", "offset": 15631314944, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.31.attn_q.bias", "offset": 15677452288, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 15677485056, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.31.attn_v.bias", "offset": 15706320896, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 15706324992, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_k.bias", "offset": 15712092160, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 15712096256, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 15715700736, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.32.attn_q.bias", "offset": 15761838080, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 15761870848, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.32.attn_v.bias", "offset": 15790706688, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 15790710784, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 15796477952, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 15796510720, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.32.ffn_gate.weight", "offset": 15978176512, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 16082255872, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 16186335232, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 16186368000, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 16186400768, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.33.ffn_gate.weight", "offset": 16368066560, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 16472145920, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 16576225280, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 16576258048, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 16576262144, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 16579866624, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.33.attn_q.bias", "offset": 16626003968, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 16626036736, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.33.attn_v.bias", "offset": 16654872576, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 16654876672, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.ffn_gate.weight", "offset": 16660643840, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.34.attn_k.bias", "offset": 16764723200, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 16764727296, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 16768331776, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.34.attn_q.bias", "offset": 16814469120, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 16814501888, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.34.attn_v.bias", "offset": 16843337728, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 16843341824, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 16849108992, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 16849141760, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.34.ffn_up.weight", "offset": 17030807552, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 17134886912, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 17134919680, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 17134952448, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.35.ffn_gate.weight", "offset": 17316618240, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 17420697600, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 17524776960, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 17524809728, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 17524813824, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 17528418304, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.35.attn_q.bias", "offset": 17574555648, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 17574588416, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.35.attn_v.bias", "offset": 17603424256, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 17603428352, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 17609195520, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 17713274880, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.36.attn_k.bias", "offset": 17817354240, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 17817358336, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 17820962816, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.36.attn_q.bias", "offset": 17867100160, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 17867132928, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.36.attn_v.bias", "offset": 17895968768, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 17895972864, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 17901740032, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 17901772800, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.36.ffn_norm.weight", "offset": 18083438592, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 18083471360, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 18083504128, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.37.ffn_gate.weight", "offset": 18265169920, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 18369249280, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 18473328640, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 18473361408, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 18473365504, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 18476969984, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.37.attn_q.bias", "offset": 18523107328, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 18523140096, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.37.attn_v.bias", "offset": 18551975936, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 18551980032, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 18557747200, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 18557779968, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.38.ffn_gate.weight", "offset": 18739445760, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 18843525120, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 18947604480, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 18947637248, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 18947641344, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 18951245824, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.38.attn_q.bias", "offset": 18997383168, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 18997415936, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.38.attn_v.bias", "offset": 19026251776, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 19026255872, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 19032023040, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 19032055808, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.39.ffn_gate.weight", "offset": 19213721600, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 19317800960, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 19421880320, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 19421913088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 19421917184, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 19425521664, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.39.attn_q.bias", "offset": 19471659008, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 19471691776, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.39.attn_v.bias", "offset": 19500527616, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 19500531712, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 19506298880, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 19506331648, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.40.ffn_gate.weight", "offset": 19687997440, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.40.ffn_up.weight", "offset": 19792076800, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.40.ffn_norm.weight", "offset": 19896156160, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 19896188928, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 19896193024, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 19899797504, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.40.attn_q.bias", "offset": 19945934848, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 19945967616, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.40.attn_v.bias", "offset": 19974803456, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 19974807552, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_k.bias", "offset": 19980574720, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 19980578816, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 19984183296, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.41.attn_q.bias", "offset": 20030320640, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 20030353408, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.41.attn_v.bias", "offset": 20059189248, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 20059193344, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 20064960512, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 20064993280, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.41.ffn_gate.weight", "offset": 20246659072, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.41.ffn_up.weight", "offset": 20350738432, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.41.ffn_norm.weight", "offset": 20454817792, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 20454850560, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 20454883328, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.42.ffn_gate.weight", "offset": 20636549120, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.42.ffn_up.weight", "offset": 20740628480, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.42.ffn_norm.weight", "offset": 20844707840, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 20844740608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 20844744704, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.42.attn_output.weight", "offset": 20848349184, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.42.attn_q.bias", "offset": 20894486528, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 20894519296, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.42.attn_v.bias", "offset": 20923355136, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 20923359232, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.ffn_gate.weight", "offset": 20929126400, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.43.attn_k.bias", "offset": 21033205760, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 21033209856, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.43.attn_output.weight", "offset": 21036814336, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.43.attn_q.bias", "offset": 21082951680, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 21082984448, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.43.attn_v.bias", "offset": 21111820288, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 21111824384, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 21117591552, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 21117624320, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.43.ffn_up.weight", "offset": 21299290112, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.43.ffn_norm.weight", "offset": 21403369472, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 21403402240, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 21403435008, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.44.ffn_gate.weight", "offset": 21585100800, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.44.ffn_up.weight", "offset": 21689180160, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.44.ffn_norm.weight", "offset": 21793259520, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 21793292288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 21793296384, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.44.attn_output.weight", "offset": 21796900864, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.44.attn_q.bias", "offset": 21843038208, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 21843070976, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.44.attn_v.bias", "offset": 21871906816, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 21871910912, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.ffn_gate.weight", "offset": 21877678080, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.45.ffn_up.weight", "offset": 21981757440, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.45.attn_k.bias", "offset": 22085836800, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 22085840896, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.45.attn_output.weight", "offset": 22089445376, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.45.attn_q.bias", "offset": 22135582720, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 22135615488, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.45.attn_v.bias", "offset": 22164451328, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 22164455424, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 22170222592, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 22170255360, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.45.ffn_norm.weight", "offset": 22351921152, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 22351953920, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 22351986688, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.46.ffn_gate.weight", "offset": 22533652480, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.46.ffn_up.weight", "offset": 22637731840, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.46.ffn_norm.weight", "offset": 22741811200, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 22741843968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 22741848064, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.46.attn_output.weight", "offset": 22745452544, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.46.attn_q.bias", "offset": 22791589888, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 22791622656, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.46.attn_v.bias", "offset": 22820458496, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 22820462592, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 22826229760, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 22826262528, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.47.ffn_gate.weight", "offset": 23007928320, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.47.ffn_up.weight", "offset": 23112007680, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.47.ffn_norm.weight", "offset": 23216087040, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 23216119808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 23216123904, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.47.attn_output.weight", "offset": 23219728384, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.47.attn_q.bias", "offset": 23265865728, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 23265898496, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.47.attn_v.bias", "offset": 23294734336, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 23294738432, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.48.attn_norm.weight", "offset": 23300505600, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 23300538368, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.48.ffn_gate.weight", "offset": 23482204160, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.48.ffn_up.weight", "offset": 23586283520, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.48.ffn_norm.weight", "offset": 23690362880, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 23690395648, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 23690399744, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.48.attn_output.weight", "offset": 23694004224, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.48.attn_q.bias", "offset": 23740141568, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 23740174336, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.48.attn_v.bias", "offset": 23769010176, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 23769014272, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.49.attn_norm.weight", "offset": 23774781440, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 23774814208, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.49.ffn_gate.weight", "offset": 23956480000, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.49.ffn_up.weight", "offset": 24060559360, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.49.ffn_norm.weight", "offset": 24164638720, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 24164671488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 24164675584, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.49.attn_output.weight", "offset": 24168280064, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.49.attn_q.bias", "offset": 24214417408, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 24214450176, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.49.attn_v.bias", "offset": 24243286016, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 24243290112, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_k.bias", "offset": 24249057280, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 24249061376, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.50.attn_output.weight", "offset": 24252665856, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.50.attn_q.bias", "offset": 24298803200, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 24298835968, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.50.attn_v.bias", "offset": 24327671808, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 24327675904, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_norm.weight", "offset": 24333443072, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 24333475840, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.50.ffn_gate.weight", "offset": 24515141632, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.50.ffn_up.weight", "offset": 24619220992, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.50.ffn_norm.weight", "offset": 24723300352, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 24723333120, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 24723365888, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.51.ffn_gate.weight", "offset": 24905031680, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.51.ffn_up.weight", "offset": 25009111040, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.51.ffn_norm.weight", "offset": 25113190400, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 25113223168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 25113227264, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.51.attn_output.weight", "offset": 25116831744, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.51.attn_q.bias", "offset": 25162969088, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 25163001856, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.51.attn_v.bias", "offset": 25191837696, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 25191841792, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.ffn_gate.weight", "offset": 25197608960, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.52.attn_k.bias", "offset": 25301688320, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 25301692416, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.52.attn_output.weight", "offset": 25305296896, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.52.attn_q.bias", "offset": 25351434240, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 25351467008, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.52.attn_v.bias", "offset": 25380302848, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 25380306944, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.attn_norm.weight", "offset": 25386074112, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 25386106880, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.52.ffn_up.weight", "offset": 25567772672, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.52.ffn_norm.weight", "offset": 25671852032, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 25671884800, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 25671917568, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.53.ffn_gate.weight", "offset": 25853583360, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.53.ffn_up.weight", "offset": 25957662720, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.53.ffn_norm.weight", "offset": 26061742080, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 26061774848, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 26061778944, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.53.attn_output.weight", "offset": 26065383424, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.53.attn_q.bias", "offset": 26111520768, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 26111553536, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.53.attn_v.bias", "offset": 26140389376, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 26140393472, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.ffn_gate.weight", "offset": 26146160640, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.54.ffn_up.weight", "offset": 26250240000, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.54.attn_k.bias", "offset": 26354319360, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 26354323456, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.54.attn_output.weight", "offset": 26357927936, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.54.attn_q.bias", "offset": 26404065280, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 26404098048, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.54.attn_v.bias", "offset": 26432933888, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 26432937984, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.attn_norm.weight", "offset": 26438705152, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 26438737920, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.54.ffn_norm.weight", "offset": 26620403712, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 26620436480, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 26620469248, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.55.ffn_gate.weight", "offset": 26802135040, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.55.ffn_up.weight", "offset": 26906214400, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.55.ffn_norm.weight", "offset": 27010293760, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 27010326528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 27010330624, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.55.attn_output.weight", "offset": 27013935104, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.55.attn_q.bias", "offset": 27060072448, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 27060105216, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.55.attn_v.bias", "offset": 27088941056, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 27088945152, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.56.attn_norm.weight", "offset": 27094712320, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 27094745088, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.56.ffn_gate.weight", "offset": 27276410880, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.56.ffn_up.weight", "offset": 27380490240, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.56.ffn_norm.weight", "offset": 27484569600, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 27484602368, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 27484606464, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.56.attn_output.weight", "offset": 27488210944, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.56.attn_q.bias", "offset": 27534348288, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 27534381056, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.56.attn_v.bias", "offset": 27563216896, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 27563220992, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.57.attn_norm.weight", "offset": 27568988160, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 27569020928, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.57.ffn_gate.weight", "offset": 27750686720, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.57.ffn_up.weight", "offset": 27854766080, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.57.ffn_norm.weight", "offset": 27958845440, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 27958878208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 27958882304, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.57.attn_output.weight", "offset": 27962486784, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.57.attn_q.bias", "offset": 28008624128, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 28008656896, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.57.attn_v.bias", "offset": 28037492736, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 28037496832, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.58.attn_norm.weight", "offset": 28043264000, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 28043296768, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.58.ffn_gate.weight", "offset": 28224962560, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.58.ffn_up.weight", "offset": 28329041920, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.58.ffn_norm.weight", "offset": 28433121280, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 28433154048, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 28433158144, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.58.attn_output.weight", "offset": 28436762624, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.58.attn_q.bias", "offset": 28482899968, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 28482932736, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.58.attn_v.bias", "offset": 28511768576, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 28511772672, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_k.bias", "offset": 28517539840, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 28517543936, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.59.attn_output.weight", "offset": 28521148416, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.59.attn_q.bias", "offset": 28567285760, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 28567318528, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.59.attn_v.bias", "offset": 28596154368, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 28596158464, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_norm.weight", "offset": 28601925632, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 28601958400, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.59.ffn_gate.weight", "offset": 28783624192, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.59.ffn_up.weight", "offset": 28887703552, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.59.ffn_norm.weight", "offset": 28991782912, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 28991815680, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 28991848448, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.60.ffn_gate.weight", "offset": 29173514240, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.60.ffn_up.weight", "offset": 29277593600, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.60.ffn_norm.weight", "offset": 29381672960, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 29381705728, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 29381709824, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.60.attn_output.weight", "offset": 29385314304, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.60.attn_q.bias", "offset": 29431451648, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 29431484416, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.60.attn_v.bias", "offset": 29460320256, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 29460324352, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.ffn_gate.weight", "offset": 29466091520, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.61.attn_k.bias", "offset": 29570170880, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 29570174976, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.61.attn_output.weight", "offset": 29573779456, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.61.attn_q.bias", "offset": 29619916800, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 29619949568, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.61.attn_v.bias", "offset": 29648785408, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 29648789504, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.attn_norm.weight", "offset": 29654556672, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 29654589440, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.61.ffn_up.weight", "offset": 29836255232, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.61.ffn_norm.weight", "offset": 29940334592, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 29940367360, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 29940400128, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.62.ffn_gate.weight", "offset": 30122065920, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.62.ffn_up.weight", "offset": 30226145280, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.62.ffn_norm.weight", "offset": 30330224640, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 30330257408, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 30330261504, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.62.attn_output.weight", "offset": 30333865984, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.62.attn_q.bias", "offset": 30380003328, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 30380036096, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.62.attn_v.bias", "offset": 30408871936, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 30408876032, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.ffn_gate.weight", "offset": 30414643200, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.63.ffn_up.weight", "offset": 30518722560, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.63.attn_k.bias", "offset": 30622801920, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 30622806016, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.63.attn_output.weight", "offset": 30626410496, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.63.attn_q.bias", "offset": 30672547840, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 30672580608, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.63.attn_v.bias", "offset": 30701416448, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 30701420544, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.attn_norm.weight", "offset": 30707187712, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 30707220480, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.63.ffn_norm.weight", "offset": 30888886272, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 30888919040, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 30888951808, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.64.ffn_gate.weight", "offset": 31070617600, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.64.ffn_up.weight", "offset": 31174696960, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.64.ffn_norm.weight", "offset": 31278776320, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 31278809088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 31278813184, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.64.attn_output.weight", "offset": 31282417664, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.64.attn_q.bias", "offset": 31328555008, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 31328587776, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.64.attn_v.bias", "offset": 31357423616, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 31357427712, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.65.attn_norm.weight", "offset": 31363194880, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 31363227648, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.65.ffn_gate.weight", "offset": 31544893440, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.65.ffn_up.weight", "offset": 31648972800, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.65.ffn_norm.weight", "offset": 31753052160, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 31753084928, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 31753089024, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.65.attn_output.weight", "offset": 31756693504, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.65.attn_q.bias", "offset": 31802830848, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 31802863616, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.65.attn_v.bias", "offset": 31831699456, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 31831703552, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.66.attn_norm.weight", "offset": 31837470720, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 31837503488, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.66.ffn_gate.weight", "offset": 32019169280, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.66.ffn_up.weight", "offset": 32123248640, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.66.ffn_norm.weight", "offset": 32227328000, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 32227360768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 32227364864, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.66.attn_output.weight", "offset": 32230969344, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.66.attn_q.bias", "offset": 32277106688, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 32277139456, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.66.attn_v.bias", "offset": 32305975296, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 32305979392, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.67.attn_norm.weight", "offset": 32311746560, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 32311779328, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.67.ffn_gate.weight", "offset": 32493445120, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.67.ffn_up.weight", "offset": 32597524480, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.67.ffn_norm.weight", "offset": 32701603840, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 32701636608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 32701640704, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.67.attn_output.weight", "offset": 32705245184, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.67.attn_q.bias", "offset": 32751382528, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 32751415296, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.67.attn_v.bias", "offset": 32780251136, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 32780255232, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_k.bias", "offset": 32786022400, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 32786026496, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.68.attn_output.weight", "offset": 32789630976, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.68.attn_q.bias", "offset": 32835768320, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 32835801088, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.68.attn_v.bias", "offset": 32864636928, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 32864641024, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_norm.weight", "offset": 32870408192, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 32870440960, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.68.ffn_gate.weight", "offset": 33052106752, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.68.ffn_up.weight", "offset": 33156186112, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.68.ffn_norm.weight", "offset": 33260265472, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 33260298240, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 33260331008, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.69.ffn_gate.weight", "offset": 33441996800, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.69.ffn_up.weight", "offset": 33546076160, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.69.ffn_norm.weight", "offset": 33650155520, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 33650188288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 33650192384, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.69.attn_output.weight", "offset": 33653796864, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.69.attn_q.bias", "offset": 33699934208, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 33699966976, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.69.attn_v.bias", "offset": 33728802816, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 33728806912, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.ffn_gate.weight", "offset": 33734574080, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.70.attn_k.bias", "offset": 33838653440, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 33838657536, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.70.attn_output.weight", "offset": 33842262016, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.70.attn_q.bias", "offset": 33888399360, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 33888432128, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.70.attn_v.bias", "offset": 33917267968, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 33917272064, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.attn_norm.weight", "offset": 33923039232, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 33923072000, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.70.ffn_up.weight", "offset": 34104737792, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.70.ffn_norm.weight", "offset": 34208817152, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 34208849920, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 34208882688, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.71.ffn_gate.weight", "offset": 34390548480, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.71.ffn_up.weight", "offset": 34494627840, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.71.ffn_norm.weight", "offset": 34598707200, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 34598739968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 34598744064, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.71.attn_output.weight", "offset": 34602348544, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.71.attn_q.bias", "offset": 34648485888, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 34648518656, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.71.attn_v.bias", "offset": 34677354496, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 34677358592, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.ffn_gate.weight", "offset": 34683125760, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.72.ffn_up.weight", "offset": 34787205120, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.72.attn_k.bias", "offset": 34891284480, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 34891288576, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.72.attn_output.weight", "offset": 34894893056, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.72.attn_q.bias", "offset": 34941030400, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 34941063168, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.72.attn_v.bias", "offset": 34969899008, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 34969903104, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.attn_norm.weight", "offset": 34975670272, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 34975703040, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.72.ffn_norm.weight", "offset": 35157368832, "shape": [ 8192 ], "size": 181665792, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 35157401600, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 35157434368, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.73.ffn_gate.weight", "offset": 35339100160, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.73.ffn_up.weight", "offset": 35443179520, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.73.ffn_norm.weight", "offset": 35547258880, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 35547291648, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 35547295744, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.73.attn_output.weight", "offset": 35550900224, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.73.attn_q.bias", "offset": 35597037568, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 35597070336, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.73.attn_v.bias", "offset": 35625906176, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 35625910272, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.74.attn_norm.weight", "offset": 35631677440, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 35631710208, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.74.ffn_gate.weight", "offset": 35813376000, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.74.ffn_up.weight", "offset": 35917455360, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.74.ffn_norm.weight", "offset": 36021534720, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 36021567488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 36021571584, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.74.attn_output.weight", "offset": 36025176064, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.74.attn_q.bias", "offset": 36071313408, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 36071346176, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.74.attn_v.bias", "offset": 36100182016, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 36100186112, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.75.attn_norm.weight", "offset": 36105953280, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 36105986048, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.75.ffn_gate.weight", "offset": 36287651840, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.75.ffn_up.weight", "offset": 36391731200, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.75.ffn_norm.weight", "offset": 36495810560, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 36495843328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 36495847424, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.75.attn_output.weight", "offset": 36499451904, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.75.attn_q.bias", "offset": 36545589248, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 36545622016, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.75.attn_v.bias", "offset": 36574457856, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 36574461952, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.76.attn_norm.weight", "offset": 36580229120, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 36580261888, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.76.ffn_gate.weight", "offset": 36761927680, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.76.ffn_up.weight", "offset": 36866007040, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.76.ffn_norm.weight", "offset": 36970086400, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 36970119168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 36970123264, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.76.attn_output.weight", "offset": 36973727744, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.76.attn_q.bias", "offset": 37019865088, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 37019897856, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.76.attn_v.bias", "offset": 37048733696, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 37048737792, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_k.bias", "offset": 37054504960, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 37054509056, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.77.attn_output.weight", "offset": 37058113536, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.77.attn_q.bias", "offset": 37104250880, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 37104283648, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.77.attn_v.bias", "offset": 37133119488, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 37133123584, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_norm.weight", "offset": 37138890752, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 37138923520, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.77.ffn_gate.weight", "offset": 37320589312, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.77.ffn_up.weight", "offset": 37424668672, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.77.ffn_norm.weight", "offset": 37528748032, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 37528780800, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 37528813568, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.78.ffn_gate.weight", "offset": 37710479360, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.78.ffn_up.weight", "offset": 37814558720, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.78.ffn_norm.weight", "offset": 37918638080, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 37918670848, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 37918674944, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.78.attn_output.weight", "offset": 37922279424, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.78.attn_q.bias", "offset": 37968416768, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 37968449536, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.78.attn_v.bias", "offset": 37997285376, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 37997289472, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.79.ffn_gate.weight", "offset": 38003056640, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.79.attn_k.bias", "offset": 38107136000, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 38107140096, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.79.attn_output.weight", "offset": 38110744576, "shape": [ 8192, 8192 ], "size": 3604480, "type": 13 }, { "name": "blk.79.attn_q.bias", "offset": 38156881920, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 38156914688, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.79.attn_v.bias", "offset": 38185750528, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 38185754624, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "output.weight", "offset": 38191521792, "shape": [ 8192, 152064 ], "size": 5767168, "type": 14 }, { "name": "blk.79.attn_norm.weight", "offset": 39213391872, "shape": [ 8192 ], "size": 1021870080, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 39213424640, "shape": [ 29568, 8192 ], "size": 32768, "type": 7 }, { "name": "blk.79.ffn_up.weight", "offset": 39395090432, "shape": [ 8192, 29568 ], "size": 181665792, "type": 11 }, { "name": "blk.79.ffn_norm.weight", "offset": 39499169792, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "output_norm.weight", "offset": 39499202560, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }