qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 8, "general.name": "Qwen2-72B", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.00001, "qwen2.block_count": 80, "qwen2.context_length": 131072, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 856424448, "shape": [ 8192, 29568 ], "size": 856424448, "type": 6 }, { "name": "blk.0.ffn_up.weight", "offset": 1022951424, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.0.attn_k.bias", "offset": 1189478400, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 1189482496, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.0.attn_output.weight", "offset": 1195249664, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.0.attn_q.bias", "offset": 1241387008, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 1241419776, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.0.attn_v.bias", "offset": 1287557120, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 1287561216, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.0.attn_norm.weight", "offset": 1293328384, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 1293361152, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.0.ffn_norm.weight", "offset": 1459888128, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1459920896, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1459953664, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.1.ffn_gate.weight", "offset": 1626480640, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.1.ffn_up.weight", "offset": 1793007616, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.1.ffn_norm.weight", "offset": 1959534592, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 1959567360, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1959571456, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.1.attn_output.weight", "offset": 1965338624, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.1.attn_q.bias", "offset": 2011475968, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 2011508736, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.1.attn_v.bias", "offset": 2057646080, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 2057650176, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.2.attn_norm.weight", "offset": 2063417344, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 2063450112, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.2.ffn_gate.weight", "offset": 2229977088, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.2.ffn_up.weight", "offset": 2396504064, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.2.ffn_norm.weight", "offset": 2563031040, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 2563063808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 2563067904, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.2.attn_output.weight", "offset": 2568835072, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.2.attn_q.bias", "offset": 2614972416, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 2615005184, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.2.attn_v.bias", "offset": 2661142528, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 2661146624, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.3.attn_norm.weight", "offset": 2666913792, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 2666946560, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.3.ffn_gate.weight", "offset": 2833473536, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.3.ffn_up.weight", "offset": 3000000512, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.3.ffn_norm.weight", "offset": 3166527488, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 3166560256, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 3166564352, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.3.attn_output.weight", "offset": 3172331520, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.3.attn_q.bias", "offset": 3218468864, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 3218501632, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.3.attn_v.bias", "offset": 3264638976, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 3264643072, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.4.attn_norm.weight", "offset": 3270410240, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 3270443008, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.4.ffn_gate.weight", "offset": 3436969984, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.4.ffn_up.weight", "offset": 3603496960, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.4.ffn_norm.weight", "offset": 3770023936, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 3770056704, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 3770060800, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.4.attn_output.weight", "offset": 3775827968, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.4.attn_q.bias", "offset": 3821965312, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 3821998080, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.4.attn_v.bias", "offset": 3868135424, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 3868139520, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.5.attn_k.bias", "offset": 3873906688, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 3873910784, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.5.attn_output.weight", "offset": 3879677952, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.5.attn_q.bias", "offset": 3925815296, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 3925848064, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.5.attn_v.bias", "offset": 3971985408, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 3971989504, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.5.attn_norm.weight", "offset": 3977756672, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 3977789440, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.5.ffn_gate.weight", "offset": 4144316416, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.5.ffn_up.weight", "offset": 4310843392, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.5.ffn_norm.weight", "offset": 4477370368, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 4477403136, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 4477435904, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.6.ffn_gate.weight", "offset": 4643962880, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.6.ffn_up.weight", "offset": 4810489856, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.6.ffn_norm.weight", "offset": 4977016832, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 4977049600, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 4977053696, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.6.attn_output.weight", "offset": 4982820864, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.6.attn_q.bias", "offset": 5028958208, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 5028990976, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.6.attn_v.bias", "offset": 5075128320, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 5075132416, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.7.ffn_gate.weight", "offset": 5080899584, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.7.attn_k.bias", "offset": 5247426560, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 5247430656, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.7.attn_output.weight", "offset": 5253197824, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.7.attn_q.bias", "offset": 5299335168, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 5299367936, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.7.attn_v.bias", "offset": 5345505280, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 5345509376, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.7.attn_norm.weight", "offset": 5351276544, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 5351309312, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.7.ffn_up.weight", "offset": 5517836288, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.7.ffn_norm.weight", "offset": 5684363264, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 5684396032, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 5684428800, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.8.ffn_gate.weight", "offset": 5850955776, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.8.ffn_up.weight", "offset": 6017482752, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.8.ffn_norm.weight", "offset": 6184009728, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 6184042496, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 6184046592, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.8.attn_output.weight", "offset": 6189813760, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.8.attn_q.bias", "offset": 6235951104, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 6235983872, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.8.attn_v.bias", "offset": 6282121216, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 6282125312, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.9.ffn_gate.weight", "offset": 6287892480, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.9.ffn_up.weight", "offset": 6454419456, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.9.attn_k.bias", "offset": 6620946432, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 6620950528, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.9.attn_output.weight", "offset": 6626717696, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.9.attn_q.bias", "offset": 6672855040, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 6672887808, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.9.attn_v.bias", "offset": 6719025152, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 6719029248, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.10.attn_norm.weight", "offset": 6724796416, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 6724829184, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.10.ffn_gate.weight", "offset": 6891356160, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.10.ffn_up.weight", "offset": 7057883136, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.10.ffn_norm.weight", "offset": 7224410112, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 7224442880, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 7224446976, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.10.attn_output.weight", "offset": 7230214144, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.10.attn_q.bias", "offset": 7276351488, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 7276384256, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.10.attn_v.bias", "offset": 7322521600, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 7322525696, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.11.attn_norm.weight", "offset": 7328292864, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 7328325632, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.11.ffn_gate.weight", "offset": 7494852608, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.11.ffn_up.weight", "offset": 7661379584, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.11.ffn_norm.weight", "offset": 7827906560, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 7827939328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 7827943424, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.11.attn_output.weight", "offset": 7833710592, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.11.attn_q.bias", "offset": 7879847936, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 7879880704, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.11.attn_v.bias", "offset": 7926018048, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 7926022144, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.9.attn_norm.weight", "offset": 7931789312, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 7931822080, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.9.ffn_norm.weight", "offset": 8098349056, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 8098381824, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 8098414592, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.12.ffn_gate.weight", "offset": 8264941568, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.12.ffn_up.weight", "offset": 8431468544, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.12.ffn_norm.weight", "offset": 8597995520, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 8598028288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 8598032384, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.12.attn_output.weight", "offset": 8603799552, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.12.attn_q.bias", "offset": 8649936896, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 8649969664, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.12.attn_v.bias", "offset": 8696107008, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 8696111104, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.13.attn_norm.weight", "offset": 8701878272, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 8701911040, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.13.ffn_gate.weight", "offset": 8868438016, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.13.ffn_up.weight", "offset": 9034964992, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.13.ffn_norm.weight", "offset": 9201491968, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 9201524736, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 9201528832, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.13.attn_output.weight", "offset": 9207296000, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.13.attn_q.bias", "offset": 9253433344, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 9253466112, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.13.attn_v.bias", "offset": 9299603456, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 9299607552, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.14.attn_k.bias", "offset": 9305374720, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 9305378816, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.14.attn_output.weight", "offset": 9311145984, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.14.attn_q.bias", "offset": 9357283328, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 9357316096, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.14.attn_v.bias", "offset": 9403453440, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 9403457536, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.14.attn_norm.weight", "offset": 9409224704, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 9409257472, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.14.ffn_gate.weight", "offset": 9575784448, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.14.ffn_up.weight", "offset": 9742311424, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.14.ffn_norm.weight", "offset": 9908838400, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 9908871168, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 9908903936, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.15.ffn_gate.weight", "offset": 10075430912, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.15.ffn_up.weight", "offset": 10241957888, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.15.ffn_norm.weight", "offset": 10408484864, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 10408517632, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 10408521728, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.15.attn_output.weight", "offset": 10414288896, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.15.attn_q.bias", "offset": 10460426240, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 10460459008, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.15.attn_v.bias", "offset": 10506596352, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 10506600448, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.16.ffn_gate.weight", "offset": 10512367616, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.16.attn_k.bias", "offset": 10678894592, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 10678898688, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.16.attn_output.weight", "offset": 10684665856, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.16.attn_q.bias", "offset": 10730803200, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 10730835968, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.16.attn_v.bias", "offset": 10776973312, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 10776977408, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.16.attn_norm.weight", "offset": 10782744576, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 10782777344, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.16.ffn_up.weight", "offset": 10949304320, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.16.ffn_norm.weight", "offset": 11115831296, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 11115864064, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 11115896832, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.17.ffn_gate.weight", "offset": 11282423808, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.17.ffn_up.weight", "offset": 11448950784, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.17.ffn_norm.weight", "offset": 11615477760, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 11615510528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 11615514624, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.17.attn_output.weight", "offset": 11621281792, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.17.attn_q.bias", "offset": 11667419136, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 11667451904, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.17.attn_v.bias", "offset": 11713589248, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 11713593344, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.18.ffn_gate.weight", "offset": 11719360512, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.18.ffn_up.weight", "offset": 11885887488, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.18.attn_k.bias", "offset": 12052414464, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 12052418560, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.18.attn_output.weight", "offset": 12058185728, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.18.attn_q.bias", "offset": 12104323072, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 12104355840, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.18.attn_v.bias", "offset": 12150493184, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 12150497280, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.18.attn_norm.weight", "offset": 12156264448, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 12156297216, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.18.ffn_norm.weight", "offset": 12322824192, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 12322856960, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 12322889728, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.19.ffn_gate.weight", "offset": 12489416704, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.19.ffn_up.weight", "offset": 12655943680, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.19.ffn_norm.weight", "offset": 12822470656, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 12822503424, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 12822507520, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.19.attn_output.weight", "offset": 12828274688, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.19.attn_q.bias", "offset": 12874412032, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 12874444800, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.19.attn_v.bias", "offset": 12920582144, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 12920586240, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.20.attn_norm.weight", "offset": 12926353408, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 12926386176, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.20.ffn_gate.weight", "offset": 13092913152, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.20.ffn_up.weight", "offset": 13259440128, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.20.ffn_norm.weight", "offset": 13425967104, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 13425999872, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 13426003968, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.20.attn_output.weight", "offset": 13431771136, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.20.attn_q.bias", "offset": 13477908480, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 13477941248, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.20.attn_v.bias", "offset": 13524078592, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 13524082688, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.21.attn_norm.weight", "offset": 13529849856, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 13529882624, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.21.ffn_gate.weight", "offset": 13696409600, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.21.ffn_up.weight", "offset": 13862936576, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.21.ffn_norm.weight", "offset": 14029463552, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 14029496320, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 14029500416, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.21.attn_output.weight", "offset": 14035267584, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.21.attn_q.bias", "offset": 14081404928, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 14081437696, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.21.attn_v.bias", "offset": 14127575040, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 14127579136, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.22.attn_norm.weight", "offset": 14133346304, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 14133379072, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.22.ffn_gate.weight", "offset": 14299906048, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.22.ffn_up.weight", "offset": 14466433024, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.22.ffn_norm.weight", "offset": 14632960000, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 14632992768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 14632996864, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.22.attn_output.weight", "offset": 14638764032, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.22.attn_q.bias", "offset": 14684901376, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 14684934144, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.22.attn_v.bias", "offset": 14731071488, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 14731075584, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.23.attn_k.bias", "offset": 14736842752, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 14736846848, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.23.attn_output.weight", "offset": 14742614016, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.23.attn_q.bias", "offset": 14788751360, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 14788784128, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.23.attn_v.bias", "offset": 14834921472, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 14834925568, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.23.attn_norm.weight", "offset": 14840692736, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 14840725504, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.23.ffn_gate.weight", "offset": 15007252480, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.23.ffn_up.weight", "offset": 15173779456, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.23.ffn_norm.weight", "offset": 15340306432, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 15340339200, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 15340371968, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.24.ffn_gate.weight", "offset": 15506898944, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.24.ffn_up.weight", "offset": 15673425920, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.24.ffn_norm.weight", "offset": 15839952896, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 15839985664, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 15839989760, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.24.attn_output.weight", "offset": 15845756928, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.24.attn_q.bias", "offset": 15891894272, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 15891927040, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.24.attn_v.bias", "offset": 15938064384, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 15938068480, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.25.ffn_gate.weight", "offset": 15943835648, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.25.attn_k.bias", "offset": 16110362624, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 16110366720, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.25.attn_output.weight", "offset": 16116133888, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.25.attn_q.bias", "offset": 16162271232, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 16162304000, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.25.attn_v.bias", "offset": 16208441344, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 16208445440, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.25.attn_norm.weight", "offset": 16214212608, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 16214245376, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.25.ffn_up.weight", "offset": 16380772352, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.25.ffn_norm.weight", "offset": 16547299328, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 16547332096, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 16547364864, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.26.ffn_gate.weight", "offset": 16713891840, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.26.ffn_up.weight", "offset": 16880418816, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.26.ffn_norm.weight", "offset": 17046945792, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 17046978560, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 17046982656, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.26.attn_output.weight", "offset": 17052749824, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.26.attn_q.bias", "offset": 17098887168, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 17098919936, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.26.attn_v.bias", "offset": 17145057280, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 17145061376, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.27.ffn_gate.weight", "offset": 17150828544, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.27.ffn_up.weight", "offset": 17317355520, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.27.attn_k.bias", "offset": 17483882496, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 17483886592, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.27.attn_output.weight", "offset": 17489653760, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.27.attn_q.bias", "offset": 17535791104, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 17535823872, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.27.attn_v.bias", "offset": 17581961216, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 17581965312, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.27.attn_norm.weight", "offset": 17587732480, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 17587765248, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.27.ffn_norm.weight", "offset": 17754292224, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 17754324992, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 17754357760, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.28.ffn_gate.weight", "offset": 17920884736, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.28.ffn_up.weight", "offset": 18087411712, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.28.ffn_norm.weight", "offset": 18253938688, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 18253971456, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 18253975552, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.28.attn_output.weight", "offset": 18259742720, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.28.attn_q.bias", "offset": 18305880064, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 18305912832, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.28.attn_v.bias", "offset": 18352050176, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 18352054272, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.29.attn_norm.weight", "offset": 18357821440, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 18357854208, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.29.ffn_gate.weight", "offset": 18524381184, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.29.ffn_up.weight", "offset": 18690908160, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.29.ffn_norm.weight", "offset": 18857435136, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 18857467904, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 18857472000, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.29.attn_output.weight", "offset": 18863239168, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.29.attn_q.bias", "offset": 18909376512, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 18909409280, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.29.attn_v.bias", "offset": 18955546624, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 18955550720, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.30.attn_norm.weight", "offset": 18961317888, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 18961350656, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.30.ffn_gate.weight", "offset": 19127877632, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.30.ffn_up.weight", "offset": 19294404608, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.30.ffn_norm.weight", "offset": 19460931584, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 19460964352, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 19460968448, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.30.attn_output.weight", "offset": 19466735616, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.30.attn_q.bias", "offset": 19512872960, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 19512905728, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.30.attn_v.bias", "offset": 19559043072, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 19559047168, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.31.attn_norm.weight", "offset": 19564814336, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 19564847104, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.31.ffn_gate.weight", "offset": 19731374080, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.31.ffn_up.weight", "offset": 19897901056, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.31.ffn_norm.weight", "offset": 20064428032, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 20064460800, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 20064464896, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.31.attn_output.weight", "offset": 20070232064, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.31.attn_q.bias", "offset": 20116369408, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 20116402176, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.31.attn_v.bias", "offset": 20162539520, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 20162543616, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.32.attn_k.bias", "offset": 20168310784, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 20168314880, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.32.attn_output.weight", "offset": 20174082048, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.32.attn_q.bias", "offset": 20220219392, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 20220252160, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.32.attn_v.bias", "offset": 20266389504, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 20266393600, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.32.attn_norm.weight", "offset": 20272160768, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 20272193536, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.32.ffn_gate.weight", "offset": 20438720512, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.32.ffn_up.weight", "offset": 20605247488, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.32.ffn_norm.weight", "offset": 20771774464, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 20771807232, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 20771840000, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.33.ffn_gate.weight", "offset": 20938366976, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.33.ffn_up.weight", "offset": 21104893952, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.33.ffn_norm.weight", "offset": 21271420928, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 21271453696, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 21271457792, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.33.attn_output.weight", "offset": 21277224960, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.33.attn_q.bias", "offset": 21323362304, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 21323395072, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.33.attn_v.bias", "offset": 21369532416, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 21369536512, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.34.ffn_gate.weight", "offset": 21375303680, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.34.attn_k.bias", "offset": 21541830656, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 21541834752, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.34.attn_output.weight", "offset": 21547601920, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.34.attn_q.bias", "offset": 21593739264, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 21593772032, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.34.attn_v.bias", "offset": 21639909376, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 21639913472, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.34.attn_norm.weight", "offset": 21645680640, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "\u0000\u0000\u0006\u0000\u0000\u0000\u0000\ufffd\ufffd\ufffd\u0006\u0000\u0000\u0000\u0012\u0000\u0000\u0000\u0000\u0000\u0000\u0000", "offset": 21645713408, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.34.ffn_up.weight", "offset": 21812240384, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.34.ffn_norm.weight", "offset": 21978767360, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 21978800128, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 21978832896, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.35.ffn_gate.weight", "offset": 22145359872, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.35.ffn_up.weight", "offset": 22311886848, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.35.ffn_norm.weight", "offset": 22478413824, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 22478446592, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 22478450688, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.35.attn_output.weight", "offset": 22484217856, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.35.attn_q.bias", "offset": 22530355200, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 22530387968, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.35.attn_v.bias", "offset": 22576525312, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 22576529408, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.36.ffn_gate.weight", "offset": 22582296576, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.36.ffn_up.weight", "offset": 22748823552, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.36.attn_k.bias", "offset": 22915350528, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 22915354624, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.36.attn_output.weight", "offset": 22921121792, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.36.attn_q.bias", "offset": 22967259136, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 22967291904, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.36.attn_v.bias", "offset": 23013429248, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 23013433344, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.36.attn_norm.weight", "offset": 23019200512, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 23019233280, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.36.ffn_norm.weight", "offset": 23185760256, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 23185793024, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 23185825792, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.37.ffn_gate.weight", "offset": 23352352768, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.37.ffn_up.weight", "offset": 23518879744, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.37.ffn_norm.weight", "offset": 23685406720, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 23685439488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 23685443584, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.37.attn_output.weight", "offset": 23691210752, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.37.attn_q.bias", "offset": 23737348096, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 23737380864, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.37.attn_v.bias", "offset": 23783518208, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 23783522304, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.38.attn_norm.weight", "offset": 23789289472, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 23789322240, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.38.ffn_gate.weight", "offset": 23955849216, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.38.ffn_up.weight", "offset": 24122376192, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.38.ffn_norm.weight", "offset": 24288903168, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 24288935936, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 24288940032, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.38.attn_output.weight", "offset": 24294707200, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.38.attn_q.bias", "offset": 24340844544, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 24340877312, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.38.attn_v.bias", "offset": 24387014656, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 24387018752, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.39.attn_norm.weight", "offset": 24392785920, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 24392818688, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.39.ffn_gate.weight", "offset": 24559345664, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.39.ffn_up.weight", "offset": 24725872640, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.39.ffn_norm.weight", "offset": 24892399616, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 24892432384, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 24892436480, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.39.attn_output.weight", "offset": 24898203648, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.39.attn_q.bias", "offset": 24944340992, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 24944373760, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.39.attn_v.bias", "offset": 24990511104, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 24990515200, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.40.attn_norm.weight", "offset": 24996282368, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 24996315136, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.40.ffn_gate.weight", "offset": 25162842112, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.40.ffn_up.weight", "offset": 25329369088, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.40.ffn_norm.weight", "offset": 25495896064, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 25495928832, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 25495932928, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.40.attn_output.weight", "offset": 25501700096, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.40.attn_q.bias", "offset": 25547837440, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 25547870208, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.40.attn_v.bias", "offset": 25594007552, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 25594011648, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.41.attn_k.bias", "offset": 25599778816, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 25599782912, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.41.attn_output.weight", "offset": 25605550080, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.41.attn_q.bias", "offset": 25651687424, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 25651720192, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.41.attn_v.bias", "offset": 25697857536, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 25697861632, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.41.attn_norm.weight", "offset": 25703628800, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 25703661568, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.41.ffn_gate.weight", "offset": 25870188544, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.41.ffn_up.weight", "offset": 26036715520, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.41.ffn_norm.weight", "offset": 26203242496, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 26203275264, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 26203308032, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.42.ffn_gate.weight", "offset": 26369835008, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.42.ffn_up.weight", "offset": 26536361984, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.42.ffn_norm.weight", "offset": 26702888960, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 26702921728, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 26702925824, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.42.attn_output.weight", "offset": 26708692992, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.42.attn_q.bias", "offset": 26754830336, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 26754863104, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.42.attn_v.bias", "offset": 26801000448, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 26801004544, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.43.ffn_gate.weight", "offset": 26806771712, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.43.attn_k.bias", "offset": 26973298688, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 26973302784, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.43.attn_output.weight", "offset": 26979069952, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.43.attn_q.bias", "offset": 27025207296, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 27025240064, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.43.attn_v.bias", "offset": 27071377408, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 27071381504, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.43.attn_norm.weight", "offset": 27077148672, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 27077181440, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.43.ffn_up.weight", "offset": 27243708416, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.43.ffn_norm.weight", "offset": 27410235392, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 27410268160, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 27410300928, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.44.ffn_gate.weight", "offset": 27576827904, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.44.ffn_up.weight", "offset": 27743354880, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.44.ffn_norm.weight", "offset": 27909881856, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 27909914624, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 27909918720, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.44.attn_output.weight", "offset": 27915685888, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.44.attn_q.bias", "offset": 27961823232, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 27961856000, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.44.attn_v.bias", "offset": 28007993344, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 28007997440, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.45.ffn_gate.weight", "offset": 28013764608, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.45.ffn_up.weight", "offset": 28180291584, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.45.attn_k.bias", "offset": 28346818560, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 28346822656, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.45.attn_output.weight", "offset": 28352589824, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.45.attn_q.bias", "offset": 28398727168, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 28398759936, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.45.attn_v.bias", "offset": 28444897280, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 28444901376, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.45.attn_norm.weight", "offset": 28450668544, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 28450701312, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.45.ffn_norm.weight", "offset": 28617228288, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 28617261056, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 28617293824, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.46.ffn_gate.weight", "offset": 28783820800, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.46.ffn_up.weight", "offset": 28950347776, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.46.ffn_norm.weight", "offset": 29116874752, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 29116907520, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 29116911616, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.46.attn_output.weight", "offset": 29122678784, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.46.attn_q.bias", "offset": 29168816128, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 29168848896, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.46.attn_v.bias", "offset": 29214986240, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 29214990336, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.47.attn_norm.weight", "offset": 29220757504, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 29220790272, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.47.ffn_gate.weight", "offset": 29387317248, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.47.ffn_up.weight", "offset": 29553844224, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.47.ffn_norm.weight", "offset": 29720371200, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 29720403968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 29720408064, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.47.attn_output.weight", "offset": 29726175232, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.47.attn_q.bias", "offset": 29772312576, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 29772345344, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.47.attn_v.bias", "offset": 29818482688, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 29818486784, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.48.attn_norm.weight", "offset": 29824253952, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 29824286720, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.48.ffn_gate.weight", "offset": 29990813696, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.48.ffn_up.weight", "offset": 30157340672, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.48.ffn_norm.weight", "offset": 30323867648, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 30323900416, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 30323904512, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.48.attn_output.weight", "offset": 30329671680, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.48.attn_q.bias", "offset": 30375809024, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 30375841792, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.48.attn_v.bias", "offset": 30421979136, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 30421983232, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.49.attn_norm.weight", "offset": 30427750400, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 30427783168, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.49.ffn_gate.weight", "offset": 30594310144, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.49.ffn_up.weight", "offset": 30760837120, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.49.ffn_norm.weight", "offset": 30927364096, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 30927396864, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 30927400960, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.49.attn_output.weight", "offset": 30933168128, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.49.attn_q.bias", "offset": 30979305472, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 30979338240, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.49.attn_v.bias", "offset": 31025475584, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 31025479680, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.50.attn_k.bias", "offset": 31031246848, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 31031250944, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.50.attn_output.weight", "offset": 31037018112, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.50.attn_q.bias", "offset": 31083155456, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 31083188224, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.50.attn_v.bias", "offset": 31129325568, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 31129329664, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.50.attn_norm.weight", "offset": 31135096832, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 31135129600, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.50.ffn_gate.weight", "offset": 31301656576, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.50.ffn_up.weight", "offset": 31468183552, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.50.ffn_norm.weight", "offset": 31634710528, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 31634743296, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 31634776064, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.51.ffn_gate.weight", "offset": 31801303040, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.51.ffn_up.weight", "offset": 31967830016, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.51.ffn_norm.weight", "offset": 32134356992, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 32134389760, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 32134393856, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.51.attn_output.weight", "offset": 32140161024, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.51.attn_q.bias", "offset": 32186298368, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 32186331136, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.51.attn_v.bias", "offset": 32232468480, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 32232472576, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.52.ffn_gate.weight", "offset": 32238239744, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.52.attn_k.bias", "offset": 32404766720, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 32404770816, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.52.attn_output.weight", "offset": 32410537984, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.52.attn_q.bias", "offset": 32456675328, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 32456708096, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.52.attn_v.bias", "offset": 32502845440, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 32502849536, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.52.attn_norm.weight", "offset": 32508616704, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 32508649472, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.52.ffn_up.weight", "offset": 32675176448, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.52.ffn_norm.weight", "offset": 32841703424, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 32841736192, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 32841768960, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.53.ffn_gate.weight", "offset": 33008295936, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.53.ffn_up.weight", "offset": 33174822912, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.53.ffn_norm.weight", "offset": 33341349888, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 33341382656, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 33341386752, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.53.attn_output.weight", "offset": 33347153920, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.53.attn_q.bias", "offset": 33393291264, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 33393324032, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.53.attn_v.bias", "offset": 33439461376, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 33439465472, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.54.ffn_gate.weight", "offset": 33445232640, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.54.ffn_up.weight", "offset": 33611759616, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.54.attn_k.bias", "offset": 33778286592, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 33778290688, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.54.attn_output.weight", "offset": 33784057856, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.54.attn_q.bias", "offset": 33830195200, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 33830227968, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.54.attn_v.bias", "offset": 33876365312, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 33876369408, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.54.attn_norm.weight", "offset": 33882136576, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 33882169344, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.54.ffn_norm.weight", "offset": 34048696320, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 34048729088, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 34048761856, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.55.ffn_gate.weight", "offset": 34215288832, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.55.ffn_up.weight", "offset": 34381815808, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.55.ffn_norm.weight", "offset": 34548342784, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 34548375552, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 34548379648, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.55.attn_output.weight", "offset": 34554146816, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.55.attn_q.bias", "offset": 34600284160, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 34600316928, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.55.attn_v.bias", "offset": 34646454272, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 34646458368, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.56.attn_norm.weight", "offset": 34652225536, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 34652258304, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.56.ffn_gate.weight", "offset": 34818785280, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.56.ffn_up.weight", "offset": 34985312256, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.56.ffn_norm.weight", "offset": 35151839232, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 35151872000, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 35151876096, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.56.attn_output.weight", "offset": 35157643264, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.56.attn_q.bias", "offset": 35203780608, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 35203813376, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.56.attn_v.bias", "offset": 35249950720, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 35249954816, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.57.attn_norm.weight", "offset": 35255721984, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 35255754752, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.57.ffn_gate.weight", "offset": 35422281728, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.57.ffn_up.weight", "offset": 35588808704, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.57.ffn_norm.weight", "offset": 35755335680, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 35755368448, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 35755372544, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.57.attn_output.weight", "offset": 35761139712, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.57.attn_q.bias", "offset": 35807277056, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 35807309824, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.57.attn_v.bias", "offset": 35853447168, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 35853451264, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.58.attn_norm.weight", "offset": 35859218432, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 35859251200, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.58.ffn_gate.weight", "offset": 36025778176, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.58.ffn_up.weight", "offset": 36192305152, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.58.ffn_norm.weight", "offset": 36358832128, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 36358864896, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 36358868992, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.58.attn_output.weight", "offset": 36364636160, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.58.attn_q.bias", "offset": 36410773504, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 36410806272, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.58.attn_v.bias", "offset": 36456943616, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 36456947712, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.59.attn_k.bias", "offset": 36462714880, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 36462718976, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.59.attn_output.weight", "offset": 36468486144, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.59.attn_q.bias", "offset": 36514623488, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 36514656256, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.59.attn_v.bias", "offset": 36560793600, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 36560797696, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.59.attn_norm.weight", "offset": 36566564864, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 36566597632, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.59.ffn_gate.weight", "offset": 36733124608, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.59.ffn_up.weight", "offset": 36899651584, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.59.ffn_norm.weight", "offset": 37066178560, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 37066211328, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 37066244096, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.60.ffn_gate.weight", "offset": 37232771072, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.60.ffn_up.weight", "offset": 37399298048, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.60.ffn_norm.weight", "offset": 37565825024, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 37565857792, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 37565861888, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.60.attn_output.weight", "offset": 37571629056, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.60.attn_q.bias", "offset": 37617766400, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 37617799168, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.60.attn_v.bias", "offset": 37663936512, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 37663940608, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.61.ffn_gate.weight", "offset": 37669707776, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.61.attn_k.bias", "offset": 37836234752, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 37836238848, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.61.attn_output.weight", "offset": 37842006016, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.61.attn_q.bias", "offset": 37888143360, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 37888176128, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.61.attn_v.bias", "offset": 37934313472, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 37934317568, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.61.attn_norm.weight", "offset": 37940084736, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 37940117504, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.61.ffn_up.weight", "offset": 38106644480, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.61.ffn_norm.weight", "offset": 38273171456, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 38273204224, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 38273236992, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.62.ffn_gate.weight", "offset": 38439763968, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.62.ffn_up.weight", "offset": 38606290944, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.62.ffn_norm.weight", "offset": 38772817920, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 38772850688, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 38772854784, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.62.attn_output.weight", "offset": 38778621952, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.62.attn_q.bias", "offset": 38824759296, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 38824792064, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.62.attn_v.bias", "offset": 38870929408, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 38870933504, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.63.ffn_gate.weight", "offset": 38876700672, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.63.ffn_up.weight", "offset": 39043227648, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.63.attn_k.bias", "offset": 39209754624, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 39209758720, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.63.attn_output.weight", "offset": 39215525888, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.63.attn_q.bias", "offset": 39261663232, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 39261696000, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.63.attn_v.bias", "offset": 39307833344, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 39307837440, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.63.attn_norm.weight", "offset": 39313604608, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 39313637376, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.63.ffn_norm.weight", "offset": 39480164352, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 39480197120, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 39480229888, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.64.ffn_gate.weight", "offset": 39646756864, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.64.ffn_up.weight", "offset": 39813283840, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.64.ffn_norm.weight", "offset": 39979810816, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 39979843584, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 39979847680, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.64.attn_output.weight", "offset": 39985614848, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.64.attn_q.bias", "offset": 40031752192, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 40031784960, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.64.attn_v.bias", "offset": 40077922304, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 40077926400, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.65.attn_norm.weight", "offset": 40083693568, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 40083726336, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.65.ffn_gate.weight", "offset": 40250253312, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.65.ffn_up.weight", "offset": 40416780288, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.65.ffn_norm.weight", "offset": 40583307264, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 40583340032, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 40583344128, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.65.attn_output.weight", "offset": 40589111296, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.65.attn_q.bias", "offset": 40635248640, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 40635281408, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.65.attn_v.bias", "offset": 40681418752, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 40681422848, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.66.attn_norm.weight", "offset": 40687190016, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 40687222784, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.66.ffn_gate.weight", "offset": 40853749760, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.66.ffn_up.weight", "offset": 41020276736, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.66.ffn_norm.weight", "offset": 41186803712, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 41186836480, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 41186840576, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.66.attn_output.weight", "offset": 41192607744, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.66.attn_q.bias", "offset": 41238745088, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 41238777856, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.66.attn_v.bias", "offset": 41284915200, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 41284919296, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.67.attn_norm.weight", "offset": 41290686464, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 41290719232, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.67.ffn_gate.weight", "offset": 41457246208, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.67.ffn_up.weight", "offset": 41623773184, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.67.ffn_norm.weight", "offset": 41790300160, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 41790332928, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 41790337024, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.67.attn_output.weight", "offset": 41796104192, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.67.attn_q.bias", "offset": 41842241536, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 41842274304, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.67.attn_v.bias", "offset": 41888411648, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 41888415744, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.68.attn_k.bias", "offset": 41894182912, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 41894187008, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.68.attn_output.weight", "offset": 41899954176, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.68.attn_q.bias", "offset": 41946091520, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 41946124288, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.68.attn_v.bias", "offset": 41992261632, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 41992265728, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.68.attn_norm.weight", "offset": 41998032896, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 41998065664, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.68.ffn_gate.weight", "offset": 42164592640, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.68.ffn_up.weight", "offset": 42331119616, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.68.ffn_norm.weight", "offset": 42497646592, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 42497679360, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 42497712128, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.69.ffn_gate.weight", "offset": 42664239104, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.69.ffn_up.weight", "offset": 42830766080, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.69.ffn_norm.weight", "offset": 42997293056, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 42997325824, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 42997329920, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.69.attn_output.weight", "offset": 43003097088, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.69.attn_q.bias", "offset": 43049234432, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 43049267200, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.69.attn_v.bias", "offset": 43095404544, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 43095408640, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.70.ffn_gate.weight", "offset": 43101175808, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.70.attn_k.bias", "offset": 43267702784, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 43267706880, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.70.attn_output.weight", "offset": 43273474048, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.70.attn_q.bias", "offset": 43319611392, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 43319644160, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.70.attn_v.bias", "offset": 43365781504, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 43365785600, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.70.attn_norm.weight", "offset": 43371552768, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 43371585536, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.70.ffn_up.weight", "offset": 43538112512, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.70.ffn_norm.weight", "offset": 43704639488, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 43704672256, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 43704705024, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.71.ffn_gate.weight", "offset": 43871232000, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.71.ffn_up.weight", "offset": 44037758976, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.71.ffn_norm.weight", "offset": 44204285952, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 44204318720, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 44204322816, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.71.attn_output.weight", "offset": 44210089984, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.71.attn_q.bias", "offset": 44256227328, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 44256260096, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.71.attn_v.bias", "offset": 44302397440, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 44302401536, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.72.ffn_gate.weight", "offset": 44308168704, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.72.ffn_up.weight", "offset": 44474695680, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.72.attn_k.bias", "offset": 44641222656, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 44641226752, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.72.attn_output.weight", "offset": 44646993920, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.72.attn_q.bias", "offset": 44693131264, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 44693164032, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.72.attn_v.bias", "offset": 44739301376, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 44739305472, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.72.attn_norm.weight", "offset": 44745072640, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 44745105408, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.72.ffn_norm.weight", "offset": 44911632384, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 44911665152, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 44911697920, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.73.ffn_gate.weight", "offset": 45078224896, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.73.ffn_up.weight", "offset": 45244751872, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.73.ffn_norm.weight", "offset": 45411278848, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 45411311616, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 45411315712, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.73.attn_output.weight", "offset": 45417082880, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.73.attn_q.bias", "offset": 45463220224, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 45463252992, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.73.attn_v.bias", "offset": 45509390336, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 45509394432, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.74.attn_norm.weight", "offset": 45515161600, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 45515194368, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.74.ffn_gate.weight", "offset": 45681721344, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.74.ffn_up.weight", "offset": 45848248320, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.74.ffn_norm.weight", "offset": 46014775296, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 46014808064, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 46014812160, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.74.attn_output.weight", "offset": 46020579328, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.74.attn_q.bias", "offset": 46066716672, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 46066749440, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.74.attn_v.bias", "offset": 46112886784, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 46112890880, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.75.attn_norm.weight", "offset": 46118658048, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 46118690816, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.75.ffn_gate.weight", "offset": 46285217792, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.75.ffn_up.weight", "offset": 46451744768, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.75.ffn_norm.weight", "offset": 46618271744, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 46618304512, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 46618308608, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.75.attn_output.weight", "offset": 46624075776, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.75.attn_q.bias", "offset": 46670213120, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 46670245888, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.75.attn_v.bias", "offset": 46716383232, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 46716387328, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.76.attn_norm.weight", "offset": 46722154496, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 46722187264, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.76.ffn_gate.weight", "offset": 46888714240, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.76.ffn_up.weight", "offset": 47055241216, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.76.ffn_norm.weight", "offset": 47221768192, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 47221800960, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 47221805056, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.76.attn_output.weight", "offset": 47227572224, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.76.attn_q.bias", "offset": 47273709568, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 47273742336, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.76.attn_v.bias", "offset": 47319879680, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 47319883776, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.77.attn_k.bias", "offset": 47325650944, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 47325655040, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.77.attn_output.weight", "offset": 47331422208, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.77.attn_q.bias", "offset": 47377559552, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 47377592320, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.77.attn_v.bias", "offset": 47423729664, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 47423733760, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.77.attn_norm.weight", "offset": 47429500928, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 47429533696, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.77.ffn_gate.weight", "offset": 47596060672, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.77.ffn_up.weight", "offset": 47762587648, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.77.ffn_norm.weight", "offset": 47929114624, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 47929147392, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 47929180160, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.78.ffn_gate.weight", "offset": 48095707136, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.78.ffn_up.weight", "offset": 48262234112, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.78.ffn_norm.weight", "offset": 48428761088, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 48428793856, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 48428797952, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.78.attn_output.weight", "offset": 48434565120, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.78.attn_q.bias", "offset": 48480702464, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 48480735232, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.78.attn_v.bias", "offset": 48526872576, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 48526876672, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.79.ffn_gate.weight", "offset": 48532643840, "shape": [ 8192, 29568 ], "size": 5767168, "type": 6 }, { "name": "blk.79.attn_k.bias", "offset": 48699170816, "shape": [ 1024 ], "size": 166526976, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 48699174912, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "blk.79.attn_output.weight", "offset": 48704942080, "shape": [ 8192, 8192 ], "size": 5767168, "type": 6 }, { "name": "blk.79.attn_q.bias", "offset": 48751079424, "shape": [ 8192 ], "size": 46137344, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 48751112192, "shape": [ 8192, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.79.attn_v.bias", "offset": 48797249536, "shape": [ 1024 ], "size": 46137344, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 48797253632, "shape": [ 8192, 1024 ], "size": 4096, "type": 6 }, { "name": "output.weight", "offset": 48803020800, "shape": [ 8192, 152064 ], "size": 5767168, "type": 14 }, { "name": "blk.79.attn_norm.weight", "offset": 49824890880, "shape": [ 8192 ], "size": 1021870080, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 49824923648, "shape": [ 29568, 8192 ], "size": 32768, "type": 6 }, { "name": "blk.79.ffn_up.weight", "offset": 49991450624, "shape": [ 8192, 29568 ], "size": 166526976, "type": 6 }, { "name": "blk.79.ffn_norm.weight", "offset": 50157977600, "shape": [ 8192 ], "size": 166526976, "type": 0 }, { "name": "output_norm.weight", "offset": 50158010368, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }