qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 7, "general.name": "Qwen2-72B", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.00001, "qwen2.block_count": 80, "qwen2.context_length": 131072, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 1323565056, "shape": [ 8192, 29568 ], "size": 1323565056, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 1580924928, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.0.attn_k.bias", "offset": 1838284800, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 1838288896, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 1847201792, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.0.attn_q.bias", "offset": 1918504960, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 1918537728, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.0.attn_v.bias", "offset": 1989840896, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 1989844992, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.0.attn_norm.weight", "offset": 1998757888, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 1998790656, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 2256150528, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 2256183296, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 2256216064, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.1.ffn_gate.weight", "offset": 2513575936, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.1.ffn_up.weight", "offset": 2770935808, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.1.ffn_norm.weight", "offset": 3028295680, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 3028328448, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 3028332544, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 3037245440, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.1.attn_q.bias", "offset": 3108548608, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 3108581376, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.1.attn_v.bias", "offset": 3179884544, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 3179888640, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 3188801536, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 3188834304, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.2.ffn_gate.weight", "offset": 3446194176, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.2.ffn_up.weight", "offset": 3703554048, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.2.ffn_norm.weight", "offset": 3960913920, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 3960946688, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 3960950784, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 3969863680, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.2.attn_q.bias", "offset": 4041166848, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 4041199616, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.2.attn_v.bias", "offset": 4112502784, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 4112506880, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.3.attn_norm.weight", "offset": 4121419776, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 4121452544, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.3.ffn_gate.weight", "offset": 4378812416, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.3.ffn_up.weight", "offset": 4636172288, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.3.ffn_norm.weight", "offset": 4893532160, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 4893564928, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 4893569024, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 4902481920, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.3.attn_q.bias", "offset": 4973785088, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 4973817856, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.3.attn_v.bias", "offset": 5045121024, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 5045125120, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 5054038016, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 5054070784, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.4.ffn_gate.weight", "offset": 5311430656, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.4.ffn_up.weight", "offset": 5568790528, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.4.ffn_norm.weight", "offset": 5826150400, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 5826183168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 5826187264, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 5835100160, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.4.attn_q.bias", "offset": 5906403328, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 5906436096, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.4.attn_v.bias", "offset": 5977739264, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 5977743360, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.5.attn_k.bias", "offset": 5986656256, "shape": [ 1024 ], "size": 8912896, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 5986660352, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 5995573248, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.5.attn_q.bias", "offset": 6066876416, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 6066909184, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.5.attn_v.bias", "offset": 6138212352, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 6138216448, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.5.attn_norm.weight", "offset": 6147129344, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 6147162112, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.5.ffn_gate.weight", "offset": 6404521984, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.5.ffn_up.weight", "offset": 6661881856, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.5.ffn_norm.weight", "offset": 6919241728, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 6919274496, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 6919307264, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.6.ffn_gate.weight", "offset": 7176667136, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.6.ffn_up.weight", "offset": 7434027008, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.6.ffn_norm.weight", "offset": 7691386880, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 7691419648, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 7691423744, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 7700336640, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.6.attn_q.bias", "offset": 7771639808, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 7771672576, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.6.attn_v.bias", "offset": 7842975744, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 7842979840, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.7.ffn_gate.weight", "offset": 7851892736, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.7.attn_k.bias", "offset": 8109252608, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 8109256704, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 8118169600, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.7.attn_q.bias", "offset": 8189472768, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 8189505536, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.7.attn_v.bias", "offset": 8260808704, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 8260812800, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.7.attn_norm.weight", "offset": 8269725696, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 8269758464, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 8527118336, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.7.ffn_norm.weight", "offset": 8784478208, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 8784510976, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 8784543744, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.8.ffn_gate.weight", "offset": 9041903616, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.8.ffn_up.weight", "offset": 9299263488, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.8.ffn_norm.weight", "offset": 9556623360, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 9556656128, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 9556660224, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 9565573120, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.8.attn_q.bias", "offset": 9636876288, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 9636909056, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.8.attn_v.bias", "offset": 9708212224, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 9708216320, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.9.ffn_gate.weight", "offset": 9717129216, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.9.ffn_up.weight", "offset": 9974489088, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.9.attn_k.bias", "offset": 10231848960, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 10231853056, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 10240765952, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.9.attn_q.bias", "offset": 10312069120, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 10312101888, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.9.attn_v.bias", "offset": 10383405056, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 10383409152, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 10392322048, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 10392354816, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.10.ffn_gate.weight", "offset": 10649714688, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.10.ffn_up.weight", "offset": 10907074560, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.10.ffn_norm.weight", "offset": 11164434432, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 11164467200, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 11164471296, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 11173384192, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.10.attn_q.bias", "offset": 11244687360, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 11244720128, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.10.attn_v.bias", "offset": 11316023296, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 11316027392, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 11324940288, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 11324973056, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.11.ffn_gate.weight", "offset": 11582332928, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.11.ffn_up.weight", "offset": 11839692800, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.11.ffn_norm.weight", "offset": 12097052672, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 12097085440, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 12097089536, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 12106002432, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.11.attn_q.bias", "offset": 12177305600, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 12177338368, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.11.attn_v.bias", "offset": 12248641536, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 12248645632, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 12257558528, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 12257591296, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.9.ffn_norm.weight", "offset": 12514951168, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 12514983936, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 12515016704, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.12.ffn_gate.weight", "offset": 12772376576, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.12.ffn_up.weight", "offset": 13029736448, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.12.ffn_norm.weight", "offset": 13287096320, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 13287129088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 13287133184, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 13296046080, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.12.attn_q.bias", "offset": 13367349248, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 13367382016, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.12.attn_v.bias", "offset": 13438685184, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 13438689280, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.13.attn_norm.weight", "offset": 13447602176, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 13447634944, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.13.ffn_gate.weight", "offset": 13704994816, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.13.ffn_up.weight", "offset": 13962354688, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.13.ffn_norm.weight", "offset": 14219714560, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 14219747328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 14219751424, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 14228664320, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.13.attn_q.bias", "offset": 14299967488, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 14300000256, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.13.attn_v.bias", "offset": 14371303424, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 14371307520, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.14.attn_k.bias", "offset": 14380220416, "shape": [ 1024 ], "size": 8912896, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 14380224512, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 14389137408, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.14.attn_q.bias", "offset": 14460440576, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 14460473344, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.14.attn_v.bias", "offset": 14531776512, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 14531780608, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 14540693504, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 14540726272, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.14.ffn_gate.weight", "offset": 14798086144, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.14.ffn_up.weight", "offset": 15055446016, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.14.ffn_norm.weight", "offset": 15312805888, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 15312838656, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 15312871424, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.15.ffn_gate.weight", "offset": 15570231296, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.15.ffn_up.weight", "offset": 15827591168, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.15.ffn_norm.weight", "offset": 16084951040, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 16084983808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 16084987904, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 16093900800, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.15.attn_q.bias", "offset": 16165203968, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 16165236736, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.15.attn_v.bias", "offset": 16236539904, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 16236544000, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.16.ffn_gate.weight", "offset": 16245456896, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.16.attn_k.bias", "offset": 16502816768, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 16502820864, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 16511733760, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.16.attn_q.bias", "offset": 16583036928, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 16583069696, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.16.attn_v.bias", "offset": 16654372864, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 16654376960, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.16.attn_norm.weight", "offset": 16663289856, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 16663322624, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.16.ffn_up.weight", "offset": 16920682496, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.16.ffn_norm.weight", "offset": 17178042368, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 17178075136, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 17178107904, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.17.ffn_gate.weight", "offset": 17435467776, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.17.ffn_up.weight", "offset": 17692827648, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.17.ffn_norm.weight", "offset": 17950187520, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 17950220288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 17950224384, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 17959137280, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.17.attn_q.bias", "offset": 18030440448, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 18030473216, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.17.attn_v.bias", "offset": 18101776384, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 18101780480, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.18.ffn_gate.weight", "offset": 18110693376, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.18.ffn_up.weight", "offset": 18368053248, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.18.attn_k.bias", "offset": 18625413120, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 18625417216, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 18634330112, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.18.attn_q.bias", "offset": 18705633280, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 18705666048, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.18.attn_v.bias", "offset": 18776969216, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 18776973312, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.18.attn_norm.weight", "offset": 18785886208, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 18785918976, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 19043278848, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 19043311616, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 19043344384, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.19.ffn_gate.weight", "offset": 19300704256, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.19.ffn_up.weight", "offset": 19558064128, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.19.ffn_norm.weight", "offset": 19815424000, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 19815456768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 19815460864, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 19824373760, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.19.attn_q.bias", "offset": 19895676928, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 19895709696, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.19.attn_v.bias", "offset": 19967012864, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 19967016960, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.20.attn_norm.weight", "offset": 19975929856, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 19975962624, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.20.ffn_gate.weight", "offset": 20233322496, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.20.ffn_up.weight", "offset": 20490682368, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.20.ffn_norm.weight", "offset": 20748042240, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 20748075008, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 20748079104, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 20756992000, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.20.attn_q.bias", "offset": 20828295168, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 20828327936, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.20.attn_v.bias", "offset": 20899631104, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 20899635200, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.21.attn_norm.weight", "offset": 20908548096, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 20908580864, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.21.ffn_gate.weight", "offset": 21165940736, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.21.ffn_up.weight", "offset": 21423300608, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.21.ffn_norm.weight", "offset": 21680660480, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 21680693248, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 21680697344, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 21689610240, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.21.attn_q.bias", "offset": 21760913408, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 21760946176, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.21.attn_v.bias", "offset": 21832249344, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 21832253440, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 21841166336, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 21841199104, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.22.ffn_gate.weight", "offset": 22098558976, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.22.ffn_up.weight", "offset": 22355918848, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.22.ffn_norm.weight", "offset": 22613278720, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 22613311488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 22613315584, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 22622228480, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.22.attn_q.bias", "offset": 22693531648, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 22693564416, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.22.attn_v.bias", "offset": 22764867584, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 22764871680, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.23.attn_k.bias", "offset": 22773784576, "shape": [ 1024 ], "size": 8912896, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 22773788672, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 22782701568, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.23.attn_q.bias", "offset": 22854004736, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 22854037504, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.23.attn_v.bias", "offset": 22925340672, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 22925344768, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.23.attn_norm.weight", "offset": 22934257664, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 22934290432, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.23.ffn_gate.weight", "offset": 23191650304, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.23.ffn_up.weight", "offset": 23449010176, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.23.ffn_norm.weight", "offset": 23706370048, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 23706402816, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 23706435584, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.24.ffn_gate.weight", "offset": 23963795456, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.24.ffn_up.weight", "offset": 24221155328, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.24.ffn_norm.weight", "offset": 24478515200, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 24478547968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 24478552064, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 24487464960, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.24.attn_q.bias", "offset": 24558768128, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 24558800896, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.24.attn_v.bias", "offset": 24630104064, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 24630108160, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.25.ffn_gate.weight", "offset": 24639021056, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.25.attn_k.bias", "offset": 24896380928, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 24896385024, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 24905297920, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.25.attn_q.bias", "offset": 24976601088, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 24976633856, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.25.attn_v.bias", "offset": 25047937024, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 25047941120, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.25.attn_norm.weight", "offset": 25056854016, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 25056886784, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.25.ffn_up.weight", "offset": 25314246656, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.25.ffn_norm.weight", "offset": 25571606528, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 25571639296, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 25571672064, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.26.ffn_gate.weight", "offset": 25829031936, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.26.ffn_up.weight", "offset": 26086391808, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.26.ffn_norm.weight", "offset": 26343751680, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 26343784448, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 26343788544, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 26352701440, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.26.attn_q.bias", "offset": 26424004608, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 26424037376, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.26.attn_v.bias", "offset": 26495340544, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 26495344640, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.27.ffn_gate.weight", "offset": 26504257536, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.27.ffn_up.weight", "offset": 26761617408, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.27.attn_k.bias", "offset": 27018977280, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 27018981376, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 27027894272, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.27.attn_q.bias", "offset": 27099197440, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 27099230208, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.27.attn_v.bias", "offset": 27170533376, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 27170537472, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.27.attn_norm.weight", "offset": 27179450368, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 27179483136, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.27.ffn_norm.weight", "offset": 27436843008, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 27436875776, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 27436908544, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.28.ffn_gate.weight", "offset": 27694268416, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.28.ffn_up.weight", "offset": 27951628288, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.28.ffn_norm.weight", "offset": 28208988160, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 28209020928, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 28209025024, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 28217937920, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.28.attn_q.bias", "offset": 28289241088, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 28289273856, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.28.attn_v.bias", "offset": 28360577024, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 28360581120, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.29.attn_norm.weight", "offset": 28369494016, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 28369526784, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.29.ffn_gate.weight", "offset": 28626886656, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.29.ffn_up.weight", "offset": 28884246528, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.29.ffn_norm.weight", "offset": 29141606400, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 29141639168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 29141643264, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 29150556160, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.29.attn_q.bias", "offset": 29221859328, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 29221892096, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.29.attn_v.bias", "offset": 29293195264, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 29293199360, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.30.attn_norm.weight", "offset": 29302112256, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 29302145024, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.30.ffn_gate.weight", "offset": 29559504896, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.30.ffn_up.weight", "offset": 29816864768, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.30.ffn_norm.weight", "offset": 30074224640, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 30074257408, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 30074261504, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 30083174400, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.30.attn_q.bias", "offset": 30154477568, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 30154510336, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.30.attn_v.bias", "offset": 30225813504, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 30225817600, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.31.attn_norm.weight", "offset": 30234730496, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 30234763264, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.31.ffn_gate.weight", "offset": 30492123136, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.31.ffn_up.weight", "offset": 30749483008, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.31.ffn_norm.weight", "offset": 31006842880, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 31006875648, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 31006879744, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 31015792640, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.31.attn_q.bias", "offset": 31087095808, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 31087128576, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.31.attn_v.bias", "offset": 31158431744, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 31158435840, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.32.attn_k.bias", "offset": 31167348736, "shape": [ 1024 ], "size": 8912896, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 31167352832, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.32.attn_output.weight", "offset": 31176265728, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.32.attn_q.bias", "offset": 31247568896, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 31247601664, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.32.attn_v.bias", "offset": 31318904832, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 31318908928, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.32.attn_norm.weight", "offset": 31327821824, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 31327854592, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.32.ffn_gate.weight", "offset": 31585214464, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.32.ffn_up.weight", "offset": 31842574336, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.32.ffn_norm.weight", "offset": 32099934208, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 32099966976, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 32099999744, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.33.ffn_gate.weight", "offset": 32357359616, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.33.ffn_up.weight", "offset": 32614719488, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.33.ffn_norm.weight", "offset": 32872079360, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 32872112128, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 32872116224, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.33.attn_output.weight", "offset": 32881029120, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.33.attn_q.bias", "offset": 32952332288, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 32952365056, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.33.attn_v.bias", "offset": 33023668224, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 33023672320, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.34.ffn_gate.weight", "offset": 33032585216, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.34.attn_k.bias", "offset": 33289945088, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 33289949184, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.34.attn_output.weight", "offset": 33298862080, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.34.attn_q.bias", "offset": 33370165248, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 33370198016, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.34.attn_v.bias", "offset": 33441501184, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 33441505280, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.34.attn_norm.weight", "offset": 33450418176, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "\u0000\u0000\b\u0000\u0000\u0000\u0000Ђz\n\u0000\u0000\u0000\u0012\u0000\u0000\u0000\u0000\u0000\u0000\u0000", "offset": 33450450944, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.34.ffn_up.weight", "offset": 33707810816, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.34.ffn_norm.weight", "offset": 33965170688, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 33965203456, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 33965236224, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.35.ffn_gate.weight", "offset": 34222596096, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.35.ffn_up.weight", "offset": 34479955968, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.35.ffn_norm.weight", "offset": 34737315840, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 34737348608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 34737352704, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.35.attn_output.weight", "offset": 34746265600, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.35.attn_q.bias", "offset": 34817568768, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 34817601536, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.35.attn_v.bias", "offset": 34888904704, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 34888908800, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.36.ffn_gate.weight", "offset": 34897821696, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.36.ffn_up.weight", "offset": 35155181568, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.36.attn_k.bias", "offset": 35412541440, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 35412545536, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.36.attn_output.weight", "offset": 35421458432, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.36.attn_q.bias", "offset": 35492761600, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 35492794368, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.36.attn_v.bias", "offset": 35564097536, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 35564101632, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.36.attn_norm.weight", "offset": 35573014528, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 35573047296, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.36.ffn_norm.weight", "offset": 35830407168, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 35830439936, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 35830472704, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.37.ffn_gate.weight", "offset": 36087832576, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.37.ffn_up.weight", "offset": 36345192448, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.37.ffn_norm.weight", "offset": 36602552320, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 36602585088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 36602589184, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.37.attn_output.weight", "offset": 36611502080, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.37.attn_q.bias", "offset": 36682805248, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 36682838016, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.37.attn_v.bias", "offset": 36754141184, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 36754145280, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.38.attn_norm.weight", "offset": 36763058176, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 36763090944, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.38.ffn_gate.weight", "offset": 37020450816, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.38.ffn_up.weight", "offset": 37277810688, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.38.ffn_norm.weight", "offset": 37535170560, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 37535203328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 37535207424, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.38.attn_output.weight", "offset": 37544120320, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.38.attn_q.bias", "offset": 37615423488, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 37615456256, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.38.attn_v.bias", "offset": 37686759424, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 37686763520, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.39.attn_norm.weight", "offset": 37695676416, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 37695709184, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.39.ffn_gate.weight", "offset": 37953069056, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.39.ffn_up.weight", "offset": 38210428928, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.39.ffn_norm.weight", "offset": 38467788800, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 38467821568, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 38467825664, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.39.attn_output.weight", "offset": 38476738560, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.39.attn_q.bias", "offset": 38548041728, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 38548074496, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.39.attn_v.bias", "offset": 38619377664, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 38619381760, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.40.attn_norm.weight", "offset": 38628294656, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 38628327424, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.40.ffn_gate.weight", "offset": 38885687296, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.40.ffn_up.weight", "offset": 39143047168, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.40.ffn_norm.weight", "offset": 39400407040, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 39400439808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 39400443904, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.40.attn_output.weight", "offset": 39409356800, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.40.attn_q.bias", "offset": 39480659968, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 39480692736, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.40.attn_v.bias", "offset": 39551995904, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 39552000000, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.41.attn_k.bias", "offset": 39560912896, "shape": [ 1024 ], "size": 8912896, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 39560916992, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.41.attn_output.weight", "offset": 39569829888, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.41.attn_q.bias", "offset": 39641133056, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 39641165824, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.41.attn_v.bias", "offset": 39712468992, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 39712473088, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.41.attn_norm.weight", "offset": 39721385984, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 39721418752, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.41.ffn_gate.weight", "offset": 39978778624, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.41.ffn_up.weight", "offset": 40236138496, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.41.ffn_norm.weight", "offset": 40493498368, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 40493531136, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 40493563904, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.42.ffn_gate.weight", "offset": 40750923776, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.42.ffn_up.weight", "offset": 41008283648, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.42.ffn_norm.weight", "offset": 41265643520, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 41265676288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 41265680384, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.42.attn_output.weight", "offset": 41274593280, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.42.attn_q.bias", "offset": 41345896448, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 41345929216, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.42.attn_v.bias", "offset": 41417232384, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 41417236480, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.43.ffn_gate.weight", "offset": 41426149376, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.43.attn_k.bias", "offset": 41683509248, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 41683513344, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.43.attn_output.weight", "offset": 41692426240, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.43.attn_q.bias", "offset": 41763729408, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 41763762176, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.43.attn_v.bias", "offset": 41835065344, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 41835069440, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.43.attn_norm.weight", "offset": 41843982336, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 41844015104, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.43.ffn_up.weight", "offset": 42101374976, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.43.ffn_norm.weight", "offset": 42358734848, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 42358767616, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 42358800384, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.44.ffn_gate.weight", "offset": 42616160256, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.44.ffn_up.weight", "offset": 42873520128, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.44.ffn_norm.weight", "offset": 43130880000, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 43130912768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 43130916864, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.44.attn_output.weight", "offset": 43139829760, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.44.attn_q.bias", "offset": 43211132928, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 43211165696, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.44.attn_v.bias", "offset": 43282468864, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 43282472960, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.45.ffn_gate.weight", "offset": 43291385856, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.45.ffn_up.weight", "offset": 43548745728, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.45.attn_k.bias", "offset": 43806105600, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 43806109696, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.45.attn_output.weight", "offset": 43815022592, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.45.attn_q.bias", "offset": 43886325760, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 43886358528, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.45.attn_v.bias", "offset": 43957661696, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 43957665792, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.45.attn_norm.weight", "offset": 43966578688, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 43966611456, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.45.ffn_norm.weight", "offset": 44223971328, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 44224004096, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 44224036864, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.46.ffn_gate.weight", "offset": 44481396736, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.46.ffn_up.weight", "offset": 44738756608, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.46.ffn_norm.weight", "offset": 44996116480, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 44996149248, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 44996153344, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.46.attn_output.weight", "offset": 45005066240, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "lk.58.attn_output.", "offset": 45076369408, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 45076402176, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.46.attn_v.bias", "offset": 45147705344, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 45147709440, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.47.attn_norm.weight", "offset": 45156622336, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 45156655104, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.47.ffn_gate.weight", "offset": 45414014976, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.47.ffn_up.weight", "offset": 45671374848, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.47.ffn_norm.weight", "offset": 45928734720, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 45928767488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 45928771584, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.47.attn_output.weight", "offset": 45937684480, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.47.attn_q.bias", "offset": 46008987648, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 46009020416, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.47.attn_v.bias", "offset": 46080323584, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 46080327680, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.48.attn_norm.weight", "offset": 46089240576, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 46089273344, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.48.ffn_gate.weight", "offset": 46346633216, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.48.ffn_up.weight", "offset": 46603993088, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.48.ffn_norm.weight", "offset": 46861352960, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 46861385728, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 46861389824, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.48.attn_output.weight", "offset": 46870302720, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.48.attn_q.bias", "offset": 46941605888, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 46941638656, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.48.attn_v.bias", "offset": 47012941824, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 47012945920, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.49.attn_norm.weight", "offset": 47021858816, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 47021891584, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.49.ffn_gate.weight", "offset": 47279251456, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.49.ffn_up.weight", "offset": 47536611328, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.49.ffn_norm.weight", "offset": 47793971200, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 47794003968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 47794008064, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.49.attn_output.weight", "offset": 47802920960, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.49.attn_q.bias", "offset": 47874224128, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 47874256896, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.49.attn_v.bias", "offset": 47945560064, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 47945564160, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.50.attn_k.bias", "offset": 47954477056, "shape": [ 1024 ], "size": 8912896, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 47954481152, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.50.attn_output.weight", "offset": 47963394048, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.50.attn_q.bias", "offset": 48034697216, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 48034729984, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.50.attn_v.bias", "offset": 48106033152, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 48106037248, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.50.attn_norm.weight", "offset": 48114950144, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 48114982912, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.50.ffn_gate.weight", "offset": 48372342784, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.50.ffn_up.weight", "offset": 48629702656, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.50.ffn_norm.weight", "offset": 48887062528, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 48887095296, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 48887128064, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.51.ffn_gate.weight", "offset": 49144487936, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.51.ffn_up.weight", "offset": 49401847808, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.51.ffn_norm.weight", "offset": 49659207680, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 49659240448, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 49659244544, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.51.attn_output.weight", "offset": 49668157440, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.51.attn_q.bias", "offset": 49739460608, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 49739493376, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.51.attn_v.bias", "offset": 49810796544, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 49810800640, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.52.ffn_gate.weight", "offset": 49819713536, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.52.attn_k.bias", "offset": 50077073408, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 50077077504, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.52.attn_output.weight", "offset": 50085990400, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.52.attn_q.bias", "offset": 50157293568, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 50157326336, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.52.attn_v.bias", "offset": 50228629504, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 50228633600, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.52.attn_norm.weight", "offset": 50237546496, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 50237579264, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.52.ffn_up.weight", "offset": 50494939136, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.52.ffn_norm.weight", "offset": 50752299008, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 50752331776, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 50752364544, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.53.ffn_gate.weight", "offset": 51009724416, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.53.ffn_up.weight", "offset": 51267084288, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.53.ffn_norm.weight", "offset": 51524444160, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 51524476928, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 51524481024, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.53.attn_output.weight", "offset": 51533393920, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.53.attn_q.bias", "offset": 51604697088, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 51604729856, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.53.attn_v.bias", "offset": 51676033024, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 51676037120, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.54.ffn_gate.weight", "offset": 51684950016, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.54.ffn_up.weight", "offset": 51942309888, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.54.attn_k.bias", "offset": 52199669760, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 52199673856, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.54.attn_output.weight", "offset": 52208586752, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.54.attn_q.bias", "offset": 52279889920, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 52279922688, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.54.attn_v.bias", "offset": 52351225856, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 52351229952, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.54.attn_norm.weight", "offset": 52360142848, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 52360175616, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.54.ffn_norm.weight", "offset": 52617535488, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 52617568256, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 52617601024, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.55.ffn_gate.weight", "offset": 52874960896, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.55.ffn_up.weight", "offset": 53132320768, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.55.ffn_norm.weight", "offset": 53389680640, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 53389713408, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 53389717504, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.55.attn_output.weight", "offset": 53398630400, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.55.attn_q.bias", "offset": 53469933568, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 53469966336, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.55.attn_v.bias", "offset": 53541269504, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 53541273600, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.56.attn_norm.weight", "offset": 53550186496, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 53550219264, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.56.ffn_gate.weight", "offset": 53807579136, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.56.ffn_up.weight", "offset": 54064939008, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.56.ffn_norm.weight", "offset": 54322298880, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 54322331648, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 54322335744, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.56.attn_output.weight", "offset": 54331248640, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.56.attn_q.bias", "offset": 54402551808, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 54402584576, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.56.attn_v.bias", "offset": 54473887744, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 54473891840, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.57.attn_norm.weight", "offset": 54482804736, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 54482837504, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.57.ffn_gate.weight", "offset": 54740197376, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.57.ffn_up.weight", "offset": 54997557248, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.57.ffn_norm.weight", "offset": 55254917120, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 55254949888, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 55254953984, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.57.attn_output.weight", "offset": 55263866880, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.57.attn_q.bias", "offset": 55335170048, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 55335202816, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.57.attn_v.bias", "offset": 55406505984, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 55406510080, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.58.attn_norm.weight", "offset": 55415422976, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 55415455744, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.58.ffn_gate.weight", "offset": 55672815616, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.58.ffn_up.weight", "offset": 55930175488, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.58.ffn_norm.weight", "offset": 56187535360, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 56187568128, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 56187572224, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.58.attn_output.weight", "offset": 56196485120, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.58.attn_q.bias", "offset": 56267788288, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 56267821056, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.58.attn_v.bias", "offset": 56339124224, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 56339128320, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.59.attn_k.bias", "offset": 56348041216, "shape": [ 1024 ], "size": 8912896, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 56348045312, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.59.attn_output.weight", "offset": 56356958208, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.59.attn_q.bias", "offset": 56428261376, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 56428294144, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.59.attn_v.bias", "offset": 56499597312, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 56499601408, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.59.attn_norm.weight", "offset": 56508514304, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 56508547072, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.59.ffn_gate.weight", "offset": 56765906944, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.59.ffn_up.weight", "offset": 57023266816, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.59.ffn_norm.weight", "offset": 57280626688, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 57280659456, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 57280692224, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.60.ffn_gate.weight", "offset": 57538052096, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.60.ffn_up.weight", "offset": 57795411968, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.60.ffn_norm.weight", "offset": 58052771840, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 58052804608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 58052808704, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.60.attn_output.weight", "offset": 58061721600, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.60.attn_q.bias", "offset": 58133024768, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 58133057536, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.60.attn_v.bias", "offset": 58204360704, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 58204364800, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.61.ffn_gate.weight", "offset": 58213277696, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.61.attn_k.bias", "offset": 58470637568, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 58470641664, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.61.attn_output.weight", "offset": 58479554560, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.61.attn_q.bias", "offset": 58550857728, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 58550890496, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.61.attn_v.bias", "offset": 58622193664, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 58622197760, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.61.attn_norm.weight", "offset": 58631110656, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 58631143424, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.61.ffn_up.weight", "offset": 58888503296, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.61.ffn_norm.weight", "offset": 59145863168, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 59145895936, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 59145928704, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.62.ffn_gate.weight", "offset": 59403288576, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.62.ffn_up.weight", "offset": 59660648448, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.62.ffn_norm.weight", "offset": 59918008320, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 59918041088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 59918045184, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.62.attn_output.weight", "offset": 59926958080, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.62.attn_q.bias", "offset": 59998261248, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 59998294016, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.62.attn_v.bias", "offset": 60069597184, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 60069601280, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.63.ffn_gate.weight", "offset": 60078514176, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.63.ffn_up.weight", "offset": 60335874048, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.63.attn_k.bias", "offset": 60593233920, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 60593238016, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.63.attn_output.weight", "offset": 60602150912, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.63.attn_q.bias", "offset": 60673454080, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 60673486848, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.63.attn_v.bias", "offset": 60744790016, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 60744794112, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.63.attn_norm.weight", "offset": 60753707008, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 60753739776, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.63.ffn_norm.weight", "offset": 61011099648, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 61011132416, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 61011165184, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.64.ffn_gate.weight", "offset": 61268525056, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.64.ffn_up.weight", "offset": 61525884928, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.64.ffn_norm.weight", "offset": 61783244800, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 61783277568, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 61783281664, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.64.attn_output.weight", "offset": 61792194560, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.64.attn_q.bias", "offset": 61863497728, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 61863530496, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.64.attn_v.bias", "offset": 61934833664, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 61934837760, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.65.attn_norm.weight", "offset": 61943750656, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 61943783424, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.65.ffn_gate.weight", "offset": 62201143296, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.65.ffn_up.weight", "offset": 62458503168, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.65.ffn_norm.weight", "offset": 62715863040, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 62715895808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 62715899904, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.65.attn_output.weight", "offset": 62724812800, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.65.attn_q.bias", "offset": 62796115968, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 62796148736, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.65.attn_v.bias", "offset": 62867451904, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 62867456000, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.66.attn_norm.weight", "offset": 62876368896, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 62876401664, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.66.ffn_gate.weight", "offset": 63133761536, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.66.ffn_up.weight", "offset": 63391121408, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.66.ffn_norm.weight", "offset": 63648481280, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 63648514048, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 63648518144, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.66.attn_output.weight", "offset": 63657431040, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.66.attn_q.bias", "offset": 63728734208, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 63728766976, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.66.attn_v.bias", "offset": 63800070144, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 63800074240, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.67.attn_norm.weight", "offset": 63808987136, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 63809019904, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.67.ffn_gate.weight", "offset": 64066379776, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.67.ffn_up.weight", "offset": 64323739648, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.67.ffn_norm.weight", "offset": 64581099520, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 64581132288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 64581136384, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.67.attn_output.weight", "offset": 64590049280, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.67.attn_q.bias", "offset": 64661352448, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 64661385216, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.67.attn_v.bias", "offset": 64732688384, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 64732692480, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.68.attn_k.bias", "offset": 64741605376, "shape": [ 1024 ], "size": 8912896, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 64741609472, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.68.attn_output.weight", "offset": 64750522368, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.68.attn_q.bias", "offset": 64821825536, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 64821858304, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.68.attn_v.bias", "offset": 64893161472, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 64893165568, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.68.attn_norm.weight", "offset": 64902078464, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 64902111232, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.68.ffn_gate.weight", "offset": 65159471104, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.68.ffn_up.weight", "offset": 65416830976, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.68.ffn_norm.weight", "offset": 65674190848, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 65674223616, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 65674256384, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.69.ffn_gate.weight", "offset": 65931616256, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.69.ffn_up.weight", "offset": 66188976128, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.69.ffn_norm.weight", "offset": 66446336000, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 66446368768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 66446372864, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.69.attn_output.weight", "offset": 66455285760, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.69.attn_q.bias", "offset": 66526588928, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 66526621696, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.69.attn_v.bias", "offset": 66597924864, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 66597928960, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.70.ffn_gate.weight", "offset": 66606841856, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.70.attn_k.bias", "offset": 66864201728, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 66864205824, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.70.attn_output.weight", "offset": 66873118720, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.70.attn_q.bias", "offset": 66944421888, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 66944454656, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "\u0014\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.70.att", "offset": 67015757824, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 67015761920, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.70.attn_norm.weight", "offset": 67024674816, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 67024707584, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.70.ffn_up.weight", "offset": 67282067456, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.70.ffn_norm.weight", "offset": 67539427328, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 67539460096, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 67539492864, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.71.ffn_gate.weight", "offset": 67796852736, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.71.ffn_up.weight", "offset": 68054212608, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.71.ffn_norm.weight", "offset": 68311572480, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 68311605248, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 68311609344, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.71.attn_output.weight", "offset": 68320522240, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.71.attn_q.bias", "offset": 68391825408, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 68391858176, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.71.attn_v.bias", "offset": 68463161344, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 68463165440, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.72.ffn_gate.weight", "offset": 68472078336, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.72.ffn_up.weight", "offset": 68729438208, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.72.attn_k.bias", "offset": 68986798080, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 68986802176, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.72.attn_output.weight", "offset": 68995715072, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.72.attn_q.bias", "offset": 69067018240, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 69067051008, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.72.attn_v.bias", "offset": 69138354176, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 69138358272, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.72.attn_norm.weight", "offset": 69147271168, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 69147303936, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.72.ffn_norm.weight", "offset": 69404663808, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 69404696576, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 69404729344, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.73.ffn_gate.weight", "offset": 69662089216, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.73.ffn_up.weight", "offset": 69919449088, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.73.ffn_norm.weight", "offset": 70176808960, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 70176841728, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 70176845824, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.73.attn_output.weight", "offset": 70185758720, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.73.attn_q.bias", "offset": 70257061888, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 70257094656, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.73.attn_v.bias", "offset": 70328397824, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 70328401920, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.74.attn_norm.weight", "offset": 70337314816, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 70337347584, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.74.ffn_gate.weight", "offset": 70594707456, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.74.ffn_up.weight", "offset": 70852067328, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.74.ffn_norm.weight", "offset": 71109427200, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 71109459968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 71109464064, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.74.attn_output.weight", "offset": 71118376960, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.74.attn_q.bias", "offset": 71189680128, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 71189712896, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.74.attn_v.bias", "offset": 71261016064, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 71261020160, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.75.attn_norm.weight", "offset": 71269933056, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 71269965824, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.75.ffn_gate.weight", "offset": 71527325696, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.75.ffn_up.weight", "offset": 71784685568, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.75.ffn_norm.weight", "offset": 72042045440, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 72042078208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 72042082304, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.75.attn_output.weight", "offset": 72050995200, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.75.attn_q.bias", "offset": 72122298368, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 72122331136, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.75.attn_v.bias", "offset": 72193634304, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 72193638400, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.76.attn_norm.weight", "offset": 72202551296, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 72202584064, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.76.ffn_gate.weight", "offset": 72459943936, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.76.ffn_up.weight", "offset": 72717303808, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.76.ffn_norm.weight", "offset": 72974663680, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 72974696448, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 72974700544, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.76.attn_output.weight", "offset": 72983613440, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.76.attn_q.bias", "offset": 73054916608, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 73054949376, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.76.attn_v.bias", "offset": 73126252544, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 73126256640, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.77.attn_k.bias", "offset": 73135169536, "shape": [ 1024 ], "size": 8912896, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 73135173632, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.77.attn_output.weight", "offset": 73144086528, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.77.attn_q.bias", "offset": 73215389696, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 73215422464, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.77.attn_v.bias", "offset": 73286725632, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 73286729728, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.77.attn_norm.weight", "offset": 73295642624, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 73295675392, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.77.ffn_gate.weight", "offset": 73553035264, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.77.ffn_up.weight", "offset": 73810395136, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.77.ffn_norm.weight", "offset": 74067755008, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 74067787776, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 74067820544, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.78.ffn_gate.weight", "offset": 74325180416, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.78.ffn_up.weight", "offset": 74582540288, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.78.ffn_norm.weight", "offset": 74839900160, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 74839932928, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 74839937024, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.78.attn_output.weight", "offset": 74848849920, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.78.attn_q.bias", "offset": 74920153088, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 74920185856, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.78.attn_v.bias", "offset": 74991489024, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 74991493120, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.79.ffn_gate.weight", "offset": 75000406016, "shape": [ 8192, 29568 ], "size": 8912896, "type": 8 }, { "name": "blk.79.attn_k.bias", "offset": 75257765888, "shape": [ 1024 ], "size": 257359872, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 75257769984, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "blk.79.attn_output.weight", "offset": 75266682880, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.79.attn_q.bias", "offset": 75337986048, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 75338018816, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.79.attn_v.bias", "offset": 75409321984, "shape": [ 1024 ], "size": 71303168, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 75409326080, "shape": [ 8192, 1024 ], "size": 4096, "type": 8 }, { "name": "output.weight", "offset": 75418238976, "shape": [ 8192, 152064 ], "size": 8912896, "type": 8 }, { "name": "blk.79.attn_norm.weight", "offset": 76741804032, "shape": [ 8192 ], "size": 1323565056, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 76741836800, "shape": [ 29568, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.79.ffn_up.weight", "offset": 76999196672, "shape": [ 8192, 29568 ], "size": 257359872, "type": 8 }, { "name": "blk.79.ffn_norm.weight", "offset": 77256556544, "shape": [ 8192 ], "size": 257359872, "type": 0 }, { "name": "output_norm.weight", "offset": 77256589312, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }