qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 11, "general.name": "Qwen2-72B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 80, "qwen2.context_length": 32768, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 535265280, "shape": [ 8192, 29568 ], "size": 535265280, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 639344640, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.0.attn_k.bias", "offset": 743424000, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 743428096, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 747032576, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.0.attn_q.bias", "offset": 775868416, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 775901184, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.0.attn_v.bias", "offset": 804737024, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 804741120, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 810508288, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 810541056, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.0.ffn_norm.weight", "offset": 946790400, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 946823168, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 946855936, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.1.ffn_gate.weight", "offset": 1083105280, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 1187184640, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 1291264000, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 1291296768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1291300864, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 1294905344, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.1.attn_q.bias", "offset": 1323741184, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 1323773952, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.1.attn_v.bias", "offset": 1352609792, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 1352613888, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 1358381056, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1358413824, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.2.ffn_gate.weight", "offset": 1494663168, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 1598742528, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 1702821888, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 1702854656, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1702858752, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 1706463232, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.2.attn_q.bias", "offset": 1735299072, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 1735331840, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.2.attn_v.bias", "offset": 1764167680, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 1764171776, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 1769938944, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1769971712, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.3.ffn_gate.weight", "offset": 1906221056, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 2010300416, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 2114379776, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 2114412544, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 2114416640, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 2118021120, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.3.attn_q.bias", "offset": 2146856960, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 2146889728, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.3.attn_v.bias", "offset": 2175725568, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 2175729664, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 2181496832, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 2181529600, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.4.ffn_gate.weight", "offset": 2317778944, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 2421858304, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 2525937664, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 2525970432, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 2525974528, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 2529579008, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.4.attn_q.bias", "offset": 2558414848, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 2558447616, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.4.attn_v.bias", "offset": 2587283456, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 2587287552, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_k.bias", "offset": 2593054720, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 2593058816, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 2596663296, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.5.attn_q.bias", "offset": 2625499136, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 2625531904, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.5.attn_v.bias", "offset": 2654367744, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 2654371840, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 2660139008, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 2660171776, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.5.ffn_gate.weight", "offset": 2796421120, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 2900500480, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 3004579840, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 3004612608, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 3004645376, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.6.ffn_gate.weight", "offset": 3140894720, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 3244974080, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 3349053440, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 3349086208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 3349090304, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 3352694784, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.6.attn_q.bias", "offset": 3381530624, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 3381563392, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.6.attn_v.bias", "offset": 3410399232, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 3410403328, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 3416170496, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.7.attn_k.bias", "offset": 3520249856, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 3520253952, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 3523858432, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.7.attn_q.bias", "offset": 3552694272, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 3552727040, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.7.attn_v.bias", "offset": 3581562880, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 3581566976, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 3587334144, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 3587366912, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.7.ffn_up.weight", "offset": 3723616256, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 3827695616, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 3827728384, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3827761152, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.8.ffn_gate.weight", "offset": 3964010496, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 4068089856, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 4172169216, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 4172201984, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 4172206080, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 4175810560, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.8.attn_q.bias", "offset": 4204646400, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 4204679168, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.8.attn_v.bias", "offset": 4233515008, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 4233519104, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 4239286272, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 4343365632, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.9.attn_k.bias", "offset": 4447444992, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 4447449088, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 4451053568, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.9.attn_q.bias", "offset": 4479889408, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 4479922176, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.9.attn_v.bias", "offset": 4508758016, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 4508762112, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 4514529280, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 4514562048, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.10.ffn_gate.weight", "offset": 4650811392, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 4754890752, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 4858970112, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 4859002880, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 4859006976, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 4862611456, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.10.attn_q.bias", "offset": 4891447296, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 4891480064, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.10.attn_v.bias", "offset": 4920315904, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 4920320000, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 4926087168, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 4926119936, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.11.ffn_gate.weight", "offset": 5062369280, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 5166448640, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 5270528000, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 5270560768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 5270564864, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 5274169344, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.11.attn_q.bias", "offset": 5303005184, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 5303037952, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.11.attn_v.bias", "offset": 5331873792, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 5331877888, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 5337645056, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 5337677824, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.9.ffn_norm.weight", "offset": 5473927168, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 5473959936, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 5473992704, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.12.ffn_gate.weight", "offset": 5610242048, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 5714321408, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 5818400768, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 5818433536, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 5818437632, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 5822042112, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.12.attn_q.bias", "offset": 5850877952, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 5850910720, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.12.attn_v.bias", "offset": 5879746560, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 5879750656, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 5885517824, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 5885550592, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.13.ffn_gate.weight", "offset": 6021799936, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 6125879296, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 6229958656, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 6229991424, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 6229995520, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 6233600000, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.13.attn_q.bias", "offset": 6262435840, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 6262468608, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.13.attn_v.bias", "offset": 6291304448, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 6291308544, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_k.bias", "offset": 6297075712, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 6297079808, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 6300684288, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.14.attn_q.bias", "offset": 6329520128, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 6329552896, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.14.attn_v.bias", "offset": 6358388736, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 6358392832, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 6364160000, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 6364192768, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.14.ffn_gate.weight", "offset": 6500442112, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 6604521472, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 6708600832, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 6708633600, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 6708666368, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.15.ffn_gate.weight", "offset": 6844915712, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 6948995072, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 7053074432, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 7053107200, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 7053111296, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 7056715776, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.15.attn_q.bias", "offset": 7085551616, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 7085584384, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.15.attn_v.bias", "offset": 7114420224, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 7114424320, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 7120191488, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.16.attn_k.bias", "offset": 7224270848, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 7224274944, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 7227879424, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.16.attn_q.bias", "offset": 7256715264, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 7256748032, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.16.attn_v.bias", "offset": 7285583872, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 7285587968, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 7291355136, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 7291387904, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.16.ffn_up.weight", "offset": 7427637248, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\ufffdS\u0002\u0000\u0000\u0000\u0014\u0000\u0000\u0000\u0000", "offset": 7531716608, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 7531749376, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 7531782144, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.17.ffn_gate.weight", "offset": 7668031488, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 7772110848, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 7876190208, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 7876222976, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 7876227072, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 7879831552, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.17.attn_q.bias", "offset": 7908667392, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 7908700160, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.17.attn_v.bias", "offset": 7937536000, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 7937540096, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 7943307264, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 8047386624, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.18.attn_k.bias", "offset": 8151465984, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 8151470080, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 8155074560, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.18.attn_q.bias", "offset": 8183910400, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 8183943168, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.18.attn_v.bias", "offset": 8212779008, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 8212783104, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 8218550272, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 8218583040, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.18.ffn_norm.weight", "offset": 8354832384, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 8354865152, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 8354897920, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.19.ffn_gate.weight", "offset": 8491147264, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 8595226624, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 8699305984, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 8699338752, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 8699342848, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 8702947328, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.19.attn_q.bias", "offset": 8731783168, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 8731815936, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.19.attn_v.bias", "offset": 8760651776, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 8760655872, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 8766423040, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 8766455808, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.20.ffn_gate.weight", "offset": 8902705152, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 9006784512, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 9110863872, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 9110896640, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 9110900736, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 9114505216, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.20.attn_q.bias", "offset": 9143341056, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 9143373824, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.20.attn_v.bias", "offset": 9172209664, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 9172213760, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 9177980928, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 9178013696, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.21.ffn_gate.weight", "offset": 9314263040, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 9418342400, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 9522421760, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 9522454528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 9522458624, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 9526063104, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.21.attn_q.bias", "offset": 9554898944, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 9554931712, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.21.attn_v.bias", "offset": 9583767552, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 9583771648, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 9589538816, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 9589571584, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.22.ffn_gate.weight", "offset": 9725820928, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 9829900288, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 9933979648, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 9934012416, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 9934016512, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 9937620992, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.22.attn_q.bias", "offset": 9966456832, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 9966489600, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.22.attn_v.bias", "offset": 9995325440, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 9995329536, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_k.bias", "offset": 10001096704, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 10001100800, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 10004705280, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.23.attn_q.bias", "offset": 10033541120, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 10033573888, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.23.attn_v.bias", "offset": 10062409728, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 10062413824, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 10068180992, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 10068213760, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.23.ffn_gate.weight", "offset": 10204463104, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 10308542464, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 10412621824, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 10412654592, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 10412687360, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.24.ffn_gate.weight", "offset": 10548936704, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 10653016064, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 10757095424, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 10757128192, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 10757132288, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 10760736768, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.24.attn_q.bias", "offset": 10789572608, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 10789605376, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.24.attn_v.bias", "offset": 10818441216, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 10818445312, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 10824212480, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.25.attn_k.bias", "offset": 10928291840, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 10928295936, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 10931900416, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.25.attn_q.bias", "offset": 10960736256, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 10960769024, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.25.attn_v.bias", "offset": 10989604864, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 10989608960, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 10995376128, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 10995408896, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.25.ffn_up.weight", "offset": 11131658240, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 11235737600, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 11235770368, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 11235803136, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.26.ffn_gate.weight", "offset": 11372052480, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 11476131840, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 11580211200, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 11580243968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 11580248064, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 11583852544, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.26.attn_q.bias", "offset": 11612688384, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 11612721152, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.26.attn_v.bias", "offset": 11641556992, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 11641561088, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 11647328256, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 11751407616, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.27.attn_k.bias", "offset": 11855486976, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 11855491072, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 11859095552, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.27.attn_q.bias", "offset": 11887931392, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 11887964160, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.27.attn_v.bias", "offset": 11916800000, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 11916804096, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 11922571264, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 11922604032, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.27.ffn_norm.weight", "offset": 12058853376, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 12058886144, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 12058918912, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.28.ffn_gate.weight", "offset": 12195168256, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 12299247616, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 12403326976, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 12403359744, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 12403363840, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 12406968320, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.28.attn_q.bias", "offset": 12435804160, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 12435836928, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.28.attn_v.bias", "offset": 12464672768, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 12464676864, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 12470444032, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 12470476800, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.29.ffn_gate.weight", "offset": 12606726144, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 12710805504, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 12814884864, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 12814917632, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 12814921728, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 12818526208, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.29.attn_q.bias", "offset": 12847362048, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 12847394816, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.29.attn_v.bias", "offset": 12876230656, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 12876234752, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 12882001920, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 12882034688, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.30.ffn_gate.weight", "offset": 13018284032, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 13122363392, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 13226442752, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 13226475520, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 13226479616, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 13230084096, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.30.attn_q.bias", "offset": 13258919936, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 13258952704, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.30.attn_v.bias", "offset": 13287788544, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 13287792640, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 13293559808, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 13293592576, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.31.ffn_gate.weight", "offset": 13429841920, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 13533921280, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 13638000640, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 13638033408, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 13638037504, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.31.attn_output.weight", "offset": 13641641984, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.31.attn_q.bias", "offset": 13670477824, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 13670510592, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.31.attn_v.bias", "offset": 13699346432, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 13699350528, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_k.bias", "offset": 13705117696, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 13705121792, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 13708726272, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.32.attn_q.bias", "offset": 13737562112, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 13737594880, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.32.attn_v.bias", "offset": 13766430720, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 13766434816, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 13772201984, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 13772234752, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.32.ffn_gate.weight", "offset": 13908484096, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 14012563456, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 14116642816, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 14116675584, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 14116708352, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.33.ffn_gate.weight", "offset": 14252957696, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 14357037056, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 14461116416, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 14461149184, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 14461153280, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 14464757760, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.33.attn_q.bias", "offset": 14493593600, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 14493626368, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.33.attn_v.bias", "offset": 14522462208, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 14522466304, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.ffn_gate.weight", "offset": 14528233472, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.34.attn_k.bias", "offset": 14632312832, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 14632316928, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 14635921408, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.34.attn_q.bias", "offset": 14664757248, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 14664790016, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.34.attn_v.bias", "offset": 14693625856, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 14693629952, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 14699397120, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 14699429888, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.34.ffn_up.weight", "offset": 14835679232, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 14939758592, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 14939791360, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 14939824128, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.35.ffn_gate.weight", "offset": 15076073472, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 15180152832, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 15284232192, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 15284264960, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 15284269056, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 15287873536, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.35.attn_q.bias", "offset": 15316709376, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 15316742144, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.35.attn_v.bias", "offset": 15345577984, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 15345582080, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 15351349248, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 15455428608, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.36.attn_k.bias", "offset": 15559507968, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 15559512064, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 15563116544, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.36.attn_q.bias", "offset": 15591952384, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 15591985152, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.36.attn_v.bias", "offset": 15620820992, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 15620825088, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 15626592256, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 15626625024, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.36.ffn_norm.weight", "offset": 15762874368, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 15762907136, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 15762939904, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.37.ffn_gate.weight", "offset": 15899189248, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 16003268608, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 16107347968, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 16107380736, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 16107384832, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 16110989312, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.37.attn_q.bias", "offset": 16139825152, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 16139857920, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.37.attn_v.bias", "offset": 16168693760, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 16168697856, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 16174465024, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 16174497792, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.38.ffn_gate.weight", "offset": 16310747136, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 16414826496, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 16518905856, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 16518938624, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 16518942720, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 16522547200, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.38.attn_q.bias", "offset": 16551383040, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 16551415808, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.38.attn_v.bias", "offset": 16580251648, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 16580255744, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 16586022912, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 16586055680, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.39.ffn_gate.weight", "offset": 16722305024, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 16826384384, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 16930463744, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 16930496512, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 16930500608, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 16934105088, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.39.attn_q.bias", "offset": 16962940928, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 16962973696, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.39.attn_v.bias", "offset": 16991809536, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 16991813632, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 16997580800, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 16997613568, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.40.ffn_gate.weight", "offset": 17133862912, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.40.ffn_up.weight", "offset": 17237942272, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.40.ffn_norm.weight", "offset": 17342021632, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 17342054400, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 17342058496, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 17345662976, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.40.attn_q.bias", "offset": 17374498816, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 17374531584, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.40.attn_v.bias", "offset": 17403367424, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 17403371520, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_k.bias", "offset": 17409138688, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 17409142784, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 17412747264, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.41.attn_q.bias", "offset": 17441583104, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 17441615872, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.41.attn_v.bias", "offset": 17470451712, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 17470455808, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 17476222976, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 17476255744, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.41.ffn_gate.weight", "offset": 17612505088, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.41.ffn_up.weight", "offset": 17716584448, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.41.ffn_norm.weight", "offset": 17820663808, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 17820696576, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 17820729344, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.42.ffn_gate.weight", "offset": 17956978688, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.42.ffn_up.weight", "offset": 18061058048, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.42.ffn_norm.weight", "offset": 18165137408, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 18165170176, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 18165174272, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.42.attn_output.weight", "offset": 18168778752, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.42.attn_q.bias", "offset": 18197614592, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 18197647360, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.42.attn_v.bias", "offset": 18226483200, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 18226487296, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.ffn_gate.weight", "offset": 18232254464, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.43.attn_k.bias", "offset": 18336333824, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 18336337920, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.43.attn_output.weight", "offset": 18339942400, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.43.attn_q.bias", "offset": 18368778240, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 18368811008, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.43.attn_v.bias", "offset": 18397646848, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 18397650944, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 18403418112, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 18403450880, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.43.ffn_up.weight", "offset": 18539700224, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.43.ffn_norm.weight", "offset": 18643779584, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 18643812352, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 18643845120, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.44.ffn_gate.weight", "offset": 18780094464, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.44.ffn_up.weight", "offset": 18884173824, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.44.ffn_norm.weight", "offset": 18988253184, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 18988285952, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 18988290048, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.44.attn_output.weight", "offset": 18991894528, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.44.attn_q.bias", "offset": 19020730368, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 19020763136, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.44.attn_v.bias", "offset": 19049598976, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 19049603072, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.ffn_gate.weight", "offset": 19055370240, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.45.ffn_up.weight", "offset": 19159449600, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.45.attn_k.bias", "offset": 19263528960, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 19263533056, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.45.attn_output.weight", "offset": 19267137536, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.45.attn_q.bias", "offset": 19295973376, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 19296006144, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.45.attn_v.bias", "offset": 19324841984, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 19324846080, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 19330613248, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 19330646016, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.45.ffn_norm.weight", "offset": 19466895360, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 19466928128, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 19466960896, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.46.ffn_gate.weight", "offset": 19603210240, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.46.ffn_up.weight", "offset": 19707289600, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.46.ffn_norm.weight", "offset": 19811368960, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 19811401728, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 19811405824, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.46.attn_output.weight", "offset": 19815010304, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.46.attn_q.bias", "offset": 19843846144, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 19843878912, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.46.attn_v.bias", "offset": 19872714752, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 19872718848, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 19878486016, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 19878518784, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.47.ffn_gate.weight", "offset": 20014768128, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.47.ffn_up.weight", "offset": 20118847488, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.47.ffn_norm.weight", "offset": 20222926848, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 20222959616, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 20222963712, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.47.attn_output.weight", "offset": 20226568192, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.47.attn_q.bias", "offset": 20255404032, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 20255436800, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.47.attn_v.bias", "offset": 20284272640, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 20284276736, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.48.attn_norm.weight", "offset": 20290043904, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 20290076672, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.48.ffn_gate.weight", "offset": 20426326016, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.48.ffn_up.weight", "offset": 20530405376, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.48.ffn_norm.weight", "offset": 20634484736, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 20634517504, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 20634521600, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.48.attn_output.weight", "offset": 20638126080, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.48.attn_q.bias", "offset": 20666961920, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 20666994688, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.48.attn_v.bias", "offset": 20695830528, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 20695834624, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.49.attn_norm.weight", "offset": 20701601792, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 20701634560, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.49.ffn_gate.weight", "offset": 20837883904, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.49.ffn_up.weight", "offset": 20941963264, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.49.ffn_norm.weight", "offset": 21046042624, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 21046075392, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 21046079488, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.49.attn_output.weight", "offset": 21049683968, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.49.attn_q.bias", "offset": 21078519808, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 21078552576, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.49.attn_v.bias", "offset": 21107388416, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 21107392512, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_k.bias", "offset": 21113159680, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 21113163776, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.50.attn_output.weight", "offset": 21116768256, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.50.attn_q.bias", "offset": 21145604096, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 21145636864, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.50.attn_v.bias", "offset": 21174472704, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 21174476800, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_norm.weight", "offset": 21180243968, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 21180276736, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.50.ffn_gate.weight", "offset": 21316526080, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.50.ffn_up.weight", "offset": 21420605440, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.50.ffn_norm.weight", "offset": 21524684800, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 21524717568, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 21524750336, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.51.ffn_gate.weight", "offset": 21660999680, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.51.ffn_up.weight", "offset": 21765079040, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.51.ffn_norm.weight", "offset": 21869158400, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 21869191168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 21869195264, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.51.attn_output.weight", "offset": 21872799744, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.51.attn_q.bias", "offset": 21901635584, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 21901668352, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.51.attn_v.bias", "offset": 21930504192, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 21930508288, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.ffn_gate.weight", "offset": 21936275456, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.52.attn_k.bias", "offset": 22040354816, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 22040358912, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.52.attn_output.weight", "offset": 22043963392, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.52.attn_q.bias", "offset": 22072799232, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 22072832000, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.52.attn_v.bias", "offset": 22101667840, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 22101671936, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.attn_norm.weight", "offset": 22107439104, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 22107471872, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.52.ffn_up.weight", "offset": 22243721216, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.52.ffn_norm.weight", "offset": 22347800576, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 22347833344, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 22347866112, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.53.ffn_gate.weight", "offset": 22484115456, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.53.ffn_up.weight", "offset": 22588194816, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.53.ffn_norm.weight", "offset": 22692274176, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 22692306944, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 22692311040, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.53.attn_output.weight", "offset": 22695915520, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.53.attn_q.bias", "offset": 22724751360, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 22724784128, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.53.attn_v.bias", "offset": 22753619968, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 22753624064, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.ffn_gate.weight", "offset": 22759391232, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.54.ffn_up.weight", "offset": 22863470592, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.54.attn_k.bias", "offset": 22967549952, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 22967554048, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.54.attn_output.weight", "offset": 22971158528, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.54.attn_q.bias", "offset": 22999994368, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 23000027136, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.54.attn_v.bias", "offset": 23028862976, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 23028867072, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.attn_norm.weight", "offset": 23034634240, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 23034667008, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.54.ffn_norm.weight", "offset": 23170916352, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 23170949120, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 23170981888, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.55.ffn_gate.weight", "offset": 23307231232, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.55.ffn_up.weight", "offset": 23411310592, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.55.ffn_norm.weight", "offset": 23515389952, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 23515422720, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 23515426816, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.55.attn_output.weight", "offset": 23519031296, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.55.attn_q.bias", "offset": 23547867136, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 23547899904, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.55.attn_v.bias", "offset": 23576735744, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 23576739840, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.56.attn_norm.weight", "offset": 23582507008, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 23582539776, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.56.ffn_gate.weight", "offset": 23718789120, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.56.ffn_up.weight", "offset": 23822868480, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.56.ffn_norm.weight", "offset": 23926947840, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 23926980608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 23926984704, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.56.attn_output.weight", "offset": 23930589184, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.56.attn_q.bias", "offset": 23959425024, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 23959457792, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.56.attn_v.bias", "offset": 23988293632, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 23988297728, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.57.attn_norm.weight", "offset": 23994064896, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 23994097664, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.57.ffn_gate.weight", "offset": 24130347008, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.57.ffn_up.weight", "offset": 24234426368, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.57.ffn_norm.weight", "offset": 24338505728, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 24338538496, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 24338542592, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.57.attn_output.weight", "offset": 24342147072, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.57.attn_q.bias", "offset": 24370982912, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 24371015680, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.57.attn_v.bias", "offset": 24399851520, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 24399855616, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.58.attn_norm.weight", "offset": 24405622784, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 24405655552, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.58.ffn_gate.weight", "offset": 24541904896, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.58.ffn_up.weight", "offset": 24645984256, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.58.ffn_norm.weight", "offset": 24750063616, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 24750096384, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 24750100480, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.58.attn_output.weight", "offset": 24753704960, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.58.attn_q.bias", "offset": 24782540800, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 24782573568, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.58.attn_v.bias", "offset": 24811409408, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 24811413504, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_k.bias", "offset": 24817180672, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 24817184768, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.59.attn_output.weight", "offset": 24820789248, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.59.attn_q.bias", "offset": 24849625088, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 24849657856, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.59.attn_v.bias", "offset": 24878493696, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 24878497792, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_norm.weight", "offset": 24884264960, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 24884297728, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.59.ffn_gate.weight", "offset": 25020547072, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.59.ffn_up.weight", "offset": 25124626432, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.59.ffn_norm.weight", "offset": 25228705792, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 25228738560, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 25228771328, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.60.ffn_gate.weight", "offset": 25365020672, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.60.ffn_up.weight", "offset": 25469100032, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.60.ffn_norm.weight", "offset": 25573179392, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 25573212160, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 25573216256, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.60.attn_output.weight", "offset": 25576820736, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.60.attn_q.bias", "offset": 25605656576, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 25605689344, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.60.attn_v.bias", "offset": 25634525184, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 25634529280, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.ffn_gate.weight", "offset": 25640296448, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.61.attn_k.bias", "offset": 25744375808, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 25744379904, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.61.attn_output.weight", "offset": 25747984384, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.61.attn_q.bias", "offset": 25776820224, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 25776852992, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.61.attn_v.bias", "offset": 25805688832, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 25805692928, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.attn_norm.weight", "offset": 25811460096, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 25811492864, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.61.ffn_up.weight", "offset": 25947742208, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.61.ffn_norm.weight", "offset": 26051821568, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 26051854336, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 26051887104, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.62.ffn_gate.weight", "offset": 26188136448, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.62.ffn_up.weight", "offset": 26292215808, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.62.ffn_norm.weight", "offset": 26396295168, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 26396327936, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 26396332032, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.62.attn_output.weight", "offset": 26399936512, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.62.attn_q.bias", "offset": 26428772352, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 26428805120, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.62.attn_v.bias", "offset": 26457640960, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 26457645056, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.ffn_gate.weight", "offset": 26463412224, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.63.ffn_up.weight", "offset": 26567491584, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.63.attn_k.bias", "offset": 26671570944, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 26671575040, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.63.attn_output.weight", "offset": 26675179520, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.63.attn_q.bias", "offset": 26704015360, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 26704048128, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.63.attn_v.bias", "offset": 26732883968, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 26732888064, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.attn_norm.weight", "offset": 26738655232, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 26738688000, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.63.ffn_norm.weight", "offset": 26874937344, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 26874970112, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 26875002880, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.64.ffn_gate.weight", "offset": 27011252224, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.64.ffn_up.weight", "offset": 27115331584, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.64.ffn_norm.weight", "offset": 27219410944, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 27219443712, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 27219447808, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.64.attn_output.weight", "offset": 27223052288, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.64.attn_q.bias", "offset": 27251888128, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 27251920896, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.64.attn_v.bias", "offset": 27280756736, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 27280760832, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.65.attn_norm.weight", "offset": 27286528000, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 27286560768, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.65.ffn_gate.weight", "offset": 27422810112, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.65.ffn_up.weight", "offset": 27526889472, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.65.ffn_norm.weight", "offset": 27630968832, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 27631001600, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 27631005696, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.65.attn_output.weight", "offset": 27634610176, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.65.attn_q.bias", "offset": 27663446016, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 27663478784, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.65.attn_v.bias", "offset": 27692314624, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 27692318720, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.66.attn_norm.weight", "offset": 27698085888, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 27698118656, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.66.ffn_gate.weight", "offset": 27834368000, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.66.ffn_up.weight", "offset": 27938447360, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.66.ffn_norm.weight", "offset": 28042526720, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 28042559488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 28042563584, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.66.attn_output.weight", "offset": 28046168064, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.66.attn_q.bias", "offset": 28075003904, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 28075036672, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.66.attn_v.bias", "offset": 28103872512, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 28103876608, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.67.attn_norm.weight", "offset": 28109643776, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 28109676544, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.67.ffn_gate.weight", "offset": 28245925888, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.67.ffn_up.weight", "offset": 28350005248, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.67.ffn_norm.weight", "offset": 28454084608, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 28454117376, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 28454121472, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.67.attn_output.weight", "offset": 28457725952, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.67.attn_q.bias", "offset": 28486561792, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 28486594560, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.67.attn_v.bias", "offset": 28515430400, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 28515434496, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_k.bias", "offset": 28521201664, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 28521205760, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.68.attn_output.weight", "offset": 28524810240, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.68.attn_q.bias", "offset": 28553646080, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 28553678848, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.68.attn_v.bias", "offset": 28582514688, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 28582518784, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_norm.weight", "offset": 28588285952, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 28588318720, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.68.ffn_gate.weight", "offset": 28724568064, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.68.ffn_up.weight", "offset": 28828647424, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.68.ffn_norm.weight", "offset": 28932726784, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 28932759552, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 28932792320, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.69.ffn_gate.weight", "offset": 29069041664, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.69.ffn_up.weight", "offset": 29173121024, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.69.ffn_norm.weight", "offset": 29277200384, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 29277233152, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 29277237248, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.69.attn_output.weight", "offset": 29280841728, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.69.attn_q.bias", "offset": 29309677568, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 29309710336, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.69.attn_v.bias", "offset": 29338546176, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 29338550272, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.ffn_gate.weight", "offset": 29344317440, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.70.attn_k.bias", "offset": 29448396800, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 29448400896, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.70.attn_output.weight", "offset": 29452005376, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.70.attn_q.bias", "offset": 29480841216, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 29480873984, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.70.attn_v.bias", "offset": 29509709824, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 29509713920, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.attn_norm.weight", "offset": 29515481088, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 29515513856, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.70.ffn_up.weight", "offset": 29651763200, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.70.ffn_norm.weight", "offset": 29755842560, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 29755875328, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 29755908096, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.71.ffn_gate.weight", "offset": 29892157440, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.71.ffn_up.weight", "offset": 29996236800, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.71.ffn_norm.weight", "offset": 30100316160, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 30100348928, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 30100353024, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.71.attn_output.weight", "offset": 30103957504, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.71.attn_q.bias", "offset": 30132793344, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 30132826112, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.71.attn_v.bias", "offset": 30161661952, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 30161666048, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.ffn_gate.weight", "offset": 30167433216, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.72.ffn_up.weight", "offset": 30271512576, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.72.attn_k.bias", "offset": 30375591936, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 30375596032, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.72.attn_output.weight", "offset": 30379200512, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.72.attn_q.bias", "offset": 30408036352, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 30408069120, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.72.attn_v.bias", "offset": 30436904960, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 30436909056, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.attn_norm.weight", "offset": 30442676224, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 30442708992, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.72.ffn_norm.weight", "offset": 30578958336, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 30578991104, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 30579023872, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.73.ffn_gate.weight", "offset": 30715273216, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.73.ffn_up.weight", "offset": 30819352576, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.73.ffn_norm.weight", "offset": 30923431936, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 30923464704, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 30923468800, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.73.attn_output.weight", "offset": 30927073280, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.73.attn_q.bias", "offset": 30955909120, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 30955941888, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.73.attn_v.bias", "offset": 30984777728, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 30984781824, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.74.attn_norm.weight", "offset": 30990548992, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 30990581760, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.74.ffn_gate.weight", "offset": 31126831104, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.74.ffn_up.weight", "offset": 31230910464, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.74.ffn_norm.weight", "offset": 31334989824, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 31335022592, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 31335026688, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.74.attn_output.weight", "offset": 31338631168, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.74.attn_q.bias", "offset": 31367467008, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 31367499776, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.74.attn_v.bias", "offset": 31396335616, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 31396339712, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.75.attn_norm.weight", "offset": 31402106880, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 31402139648, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.75.ffn_gate.weight", "offset": 31538388992, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.75.ffn_up.weight", "offset": 31642468352, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.75.ffn_norm.weight", "offset": 31746547712, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 31746580480, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 31746584576, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.75.attn_output.weight", "offset": 31750189056, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.75.attn_q.bias", "offset": 31779024896, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 31779057664, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.75.attn_v.bias", "offset": 31807893504, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 31807897600, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.76.attn_norm.weight", "offset": 31813664768, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 31813697536, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.76.ffn_gate.weight", "offset": 31949946880, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.76.ffn_up.weight", "offset": 32054026240, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.76.ffn_norm.weight", "offset": 32158105600, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 32158138368, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 32158142464, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.76.attn_output.weight", "offset": 32161746944, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.76.attn_q.bias", "offset": 32190582784, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 32190615552, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.76.attn_v.bias", "offset": 32219451392, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 32219455488, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_k.bias", "offset": 32225222656, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 32225226752, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.77.attn_output.weight", "offset": 32228831232, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.77.attn_q.bias", "offset": 32257667072, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 32257699840, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.77.attn_v.bias", "offset": 32286535680, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 32286539776, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_norm.weight", "offset": 32292306944, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 32292339712, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.77.ffn_gate.weight", "offset": 32428589056, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.77.ffn_up.weight", "offset": 32532668416, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.77.ffn_norm.weight", "offset": 32636747776, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 32636780544, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 32636813312, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.78.ffn_gate.weight", "offset": 32773062656, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.78.ffn_up.weight", "offset": 32877142016, "shape": [ 8192, 29568 ], "size": 104079360, "type": 11 }, { "name": "blk.78.ffn_norm.weight", "offset": 32981221376, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 32981254144, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 32981258240, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.78.attn_output.weight", "offset": 32984862720, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.78.attn_q.bias", "offset": 33013698560, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 33013731328, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.78.attn_v.bias", "offset": 33042567168, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 33042571264, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.79.ffn_gate.weight", "offset": 33048338432, "shape": [ 8192, 29568 ], "size": 5767168, "type": 11 }, { "name": "blk.79.attn_k.bias", "offset": 33152417792, "shape": [ 1024 ], "size": 104079360, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 33152421888, "shape": [ 8192, 1024 ], "size": 4096, "type": 11 }, { "name": "blk.79.attn_output.weight", "offset": 33156026368, "shape": [ 8192, 8192 ], "size": 3604480, "type": 11 }, { "name": "blk.79.attn_q.bias", "offset": 33184862208, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 33184894976, "shape": [ 8192, 8192 ], "size": 32768, "type": 11 }, { "name": "blk.79.attn_v.bias", "offset": 33213730816, "shape": [ 1024 ], "size": 28835840, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 33213734912, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "output.weight", "offset": 33219502080, "shape": [ 8192, 152064 ], "size": 5767168, "type": 14 }, { "name": "blk.79.attn_norm.weight", "offset": 34241372160, "shape": [ 8192 ], "size": 1021870080, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 34241404928, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.79.ffn_up.weight", "offset": 34377654272, "shape": [ 8192, 29568 ], "size": 136249344, "type": 11 }, { "name": "blk.79.ffn_norm.weight", "offset": 34481733632, "shape": [ 8192 ], "size": 104079360, "type": 0 }, { "name": "output_norm.weight", "offset": 34481766400, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }