qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 1, "general.name": "Qwen2-72B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 80, "qwen2.context_length": 32768, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 2491416576, "shape": [ 8192, 29568 ], "size": 2491416576, "type": 1 }, { "name": "blk.0.ffn_up.weight", "offset": 2975858688, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.0.attn_k.bias", "offset": 3460300800, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 3460304896, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.0.attn_output.weight", "offset": 3477082112, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.0.attn_q.bias", "offset": 3611299840, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 3611332608, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.0.attn_v.bias", "offset": 3745550336, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 3745554432, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.0.attn_norm.weight", "offset": 3762331648, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 3762364416, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.0.ffn_norm.weight", "offset": 4246806528, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 4246839296, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 4246872064, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.1.ffn_gate.weight", "offset": 4731314176, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.1.ffn_up.weight", "offset": 5215756288, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.1.ffn_norm.weight", "offset": 5700198400, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 5700231168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 5700235264, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.1.attn_output.weight", "offset": 5717012480, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.1.attn_q.bias", "offset": 5851230208, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 5851262976, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.1.attn_v.bias", "offset": 5985480704, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 5985484800, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 6002262016, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 6002294784, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.2.ffn_gate.weight", "offset": 6486736896, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.2.ffn_up.weight", "offset": 6971179008, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.2.ffn_norm.weight", "offset": 7455621120, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 7455653888, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 7455657984, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.2.attn_output.weight", "offset": 7472435200, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.2.attn_q.bias", "offset": 7606652928, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 7606685696, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.2.attn_v.bias", "offset": 7740903424, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 7740907520, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.3.attn_norm.weight", "offset": 7757684736, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 7757717504, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.3.ffn_gate.weight", "offset": 8242159616, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.3.ffn_up.weight", "offset": 8726601728, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.3.ffn_norm.weight", "offset": 9211043840, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 9211076608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 9211080704, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.3.attn_output.weight", "offset": 9227857920, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.3.attn_q.bias", "offset": 9362075648, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 9362108416, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.3.attn_v.bias", "offset": 9496326144, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 9496330240, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 9513107456, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 9513140224, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.4.ffn_gate.weight", "offset": 9997582336, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.4.ffn_up.weight", "offset": 10482024448, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.4.ffn_norm.weight", "offset": 10966466560, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 10966499328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 10966503424, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.4.attn_output.weight", "offset": 10983280640, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.4.attn_q.bias", "offset": 11117498368, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 11117531136, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.4.attn_v.bias", "offset": 11251748864, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "\u0000\u0000\u0016\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.16.ff", "offset": 11251752960, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.5.attn_k.bias", "offset": 11268530176, "shape": [ 1024 ], "size": 16777216, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 11268534272, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.5.attn_output.weight", "offset": 11285311488, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.5.attn_q.bias", "offset": 11419529216, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 11419561984, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.5.attn_v.bias", "offset": 11553779712, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 11553783808, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.5.attn_norm.weight", "offset": 11570561024, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 11570593792, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.5.ffn_gate.weight", "offset": 12055035904, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.5.ffn_up.weight", "offset": 12539478016, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.5.ffn_norm.weight", "offset": 13023920128, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 13023952896, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 13023985664, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.6.ffn_gate.weight", "offset": 13508427776, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.6.ffn_up.weight", "offset": 13992869888, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.6.ffn_norm.weight", "offset": 14477312000, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 14477344768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 14477348864, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.6.attn_output.weight", "offset": 14494126080, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.6.attn_q.bias", "offset": 14628343808, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 14628376576, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.6.attn_v.bias", "offset": 14762594304, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 14762598400, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.7.ffn_gate.weight", "offset": 14779375616, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.7.attn_k.bias", "offset": 15263817728, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 15263821824, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.7.attn_output.weight", "offset": 15280599040, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.7.attn_q.bias", "offset": 15414816768, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 15414849536, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.7.attn_v.bias", "offset": 15549067264, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 15549071360, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.7.attn_norm.weight", "offset": 15565848576, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 15565881344, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.7.ffn_up.weight", "offset": 16050323456, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.7.ffn_norm.weight", "offset": 16534765568, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 16534798336, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 16534831104, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.8.ffn_gate.weight", "offset": 17019273216, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.8.ffn_up.weight", "offset": 17503715328, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.8.ffn_norm.weight", "offset": 17988157440, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 17988190208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 17988194304, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.8.attn_output.weight", "offset": 18004971520, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.8.attn_q.bias", "offset": 18139189248, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 18139222016, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.8.attn_v.bias", "offset": 18273439744, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 18273443840, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.9.ffn_gate.weight", "offset": 18290221056, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.9.ffn_up.weight", "offset": 18774663168, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.9.attn_k.bias", "offset": 19259105280, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 19259109376, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.9.attn_output.weight", "offset": 19275886592, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.9.attn_q.bias", "offset": 19410104320, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 19410137088, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.9.attn_v.bias", "offset": 19544354816, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 19544358912, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.10.attn_norm.weight", "offset": 19561136128, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 19561168896, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.10.ffn_gate.weight", "offset": 20045611008, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.10.ffn_up.weight", "offset": 20530053120, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.10.ffn_norm.weight", "offset": 21014495232, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 21014528000, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 21014532096, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.10.attn_output.weight", "offset": 21031309312, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.10.attn_q.bias", "offset": 21165527040, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 21165559808, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.10.attn_v.bias", "offset": 21299777536, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 21299781632, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.11.attn_norm.weight", "offset": 21316558848, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 21316591616, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.11.ffn_gate.weight", "offset": 21801033728, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.11.ffn_up.weight", "offset": 22285475840, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.11.ffn_norm.weight", "offset": 22769917952, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 22769950720, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 22769954816, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.11.attn_output.weight", "offset": 22786732032, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.11.attn_q.bias", "offset": 22920949760, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 22920982528, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.11.attn_v.bias", "offset": 23055200256, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 23055204352, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 23071981568, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 23072014336, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.9.ffn_norm.weight", "offset": 23556456448, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 23556489216, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 23556521984, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.12.ffn_gate.weight", "offset": 24040964096, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.12.ffn_up.weight", "offset": 24525406208, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.12.ffn_norm.weight", "offset": 25009848320, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 25009881088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 25009885184, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.12.attn_output.weight", "offset": 25026662400, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.12.attn_q.bias", "offset": 25160880128, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 25160912896, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.12.attn_v.bias", "offset": 25295130624, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 25295134720, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.13.attn_norm.weight", "offset": 25311911936, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 25311944704, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.13.ffn_gate.weight", "offset": 25796386816, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.13.ffn_up.weight", "offset": 26280828928, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.13.ffn_norm.weight", "offset": 26765271040, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 26765303808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 26765307904, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.13.attn_output.weight", "offset": 26782085120, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.13.attn_q.bias", "offset": 26916302848, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 26916335616, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.13.attn_v.bias", "offset": 27050553344, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 27050557440, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.14.attn_k.bias", "offset": 27067334656, "shape": [ 1024 ], "size": 16777216, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 27067338752, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.14.attn_output.weight", "offset": 27084115968, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.14.attn_q.bias", "offset": 27218333696, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 27218366464, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.14.attn_v.bias", "offset": 27352584192, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 27352588288, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 27369365504, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 27369398272, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.14.ffn_gate.weight", "offset": 27853840384, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.14.ffn_up.weight", "offset": 28338282496, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.14.ffn_norm.weight", "offset": 28822724608, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 28822757376, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 28822790144, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.15.ffn_gate.weight", "offset": 29307232256, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.15.ffn_up.weight", "offset": 29791674368, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.15.ffn_norm.weight", "offset": 30276116480, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 30276149248, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 30276153344, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.15.attn_output.weight", "offset": 30292930560, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.15.attn_q.bias", "offset": 30427148288, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 30427181056, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.15.attn_v.bias", "offset": 30561398784, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 30561402880, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.16.ffn_gate.weight", "offset": 30578180096, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.16.attn_k.bias", "offset": 31062622208, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 31062626304, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.16.attn_output.weight", "offset": 31079403520, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.16.attn_q.bias", "offset": 31213621248, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 31213654016, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.16.attn_v.bias", "offset": 31347871744, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 31347875840, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.16.attn_norm.weight", "offset": 31364653056, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 31364685824, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.16.ffn_up.weight", "offset": 31849127936, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "\u0012\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.28.attn_v.", "offset": 32333570048, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 32333602816, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 32333635584, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.17.ffn_gate.weight", "offset": 32818077696, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.17.ffn_up.weight", "offset": 33302519808, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.17.ffn_norm.weight", "offset": 33786961920, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 33786994688, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 33786998784, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.17.attn_output.weight", "offset": 33803776000, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.17.attn_q.bias", "offset": 33937993728, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 33938026496, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.17.attn_v.bias", "offset": 34072244224, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 34072248320, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.18.ffn_gate.weight", "offset": 34089025536, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.18.ffn_up.weight", "offset": 34573467648, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.18.attn_k.bias", "offset": 35057909760, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 35057913856, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.18.attn_output.weight", "offset": 35074691072, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.18.attn_q.bias", "offset": 35208908800, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 35208941568, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.18.attn_v.bias", "offset": 35343159296, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 35343163392, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.18.attn_norm.weight", "offset": 35359940608, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 35359973376, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.18.ffn_norm.weight", "offset": 35844415488, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 35844448256, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 35844481024, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.19.ffn_gate.weight", "offset": 36328923136, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.19.ffn_up.weight", "offset": 36813365248, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.19.ffn_norm.weight", "offset": 37297807360, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 37297840128, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 37297844224, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.19.attn_output.weight", "offset": 37314621440, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.19.attn_q.bias", "offset": 37448839168, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 37448871936, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.19.attn_v.bias", "offset": 37583089664, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 37583093760, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.20.attn_norm.weight", "offset": 37599870976, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 37599903744, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.20.ffn_gate.weight", "offset": 38084345856, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.20.ffn_up.weight", "offset": 38568787968, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.20.ffn_norm.weight", "offset": 39053230080, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 39053262848, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 39053266944, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.20.attn_output.weight", "offset": 39070044160, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.20.attn_q.bias", "offset": 39204261888, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 39204294656, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.20.attn_v.bias", "offset": 39338512384, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 39338516480, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 39355293696, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 39355326464, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.21.ffn_gate.weight", "offset": 39839768576, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.21.ffn_up.weight", "offset": 40324210688, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.21.ffn_norm.weight", "offset": 40808652800, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 40808685568, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 40808689664, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.21.attn_output.weight", "offset": 40825466880, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.21.attn_q.bias", "offset": 40959684608, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 40959717376, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.21.attn_v.bias", "offset": 41093935104, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 41093939200, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 41110716416, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 41110749184, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.22.ffn_gate.weight", "offset": 41595191296, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.22.ffn_up.weight", "offset": 42079633408, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.22.ffn_norm.weight", "offset": 42564075520, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 42564108288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 42564112384, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.22.attn_output.weight", "offset": 42580889600, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.22.attn_q.bias", "offset": 42715107328, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 42715140096, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.22.attn_v.bias", "offset": 42849357824, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 42849361920, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.23.attn_k.bias", "offset": 42866139136, "shape": [ 1024 ], "size": 16777216, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 42866143232, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.23.attn_output.weight", "offset": 42882920448, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.23.attn_q.bias", "offset": 43017138176, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 43017170944, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.23.attn_v.bias", "offset": 43151388672, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 43151392768, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 43168169984, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 43168202752, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.23.ffn_gate.weight", "offset": 43652644864, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.23.ffn_up.weight", "offset": 44137086976, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.23.ffn_norm.weight", "offset": 44621529088, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 44621561856, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 44621594624, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.24.ffn_gate.weight", "offset": 45106036736, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.24.ffn_up.weight", "offset": 45590478848, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.24.ffn_norm.weight", "offset": 46074920960, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 46074953728, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 46074957824, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.24.attn_output.weight", "offset": 46091735040, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.24.attn_q.bias", "offset": 46225952768, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 46225985536, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.24.attn_v.bias", "offset": 46360203264, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 46360207360, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.25.ffn_gate.weight", "offset": 46376984576, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.25.attn_k.bias", "offset": 46861426688, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 46861430784, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.25.attn_output.weight", "offset": 46878208000, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.25.attn_q.bias", "offset": 47012425728, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 47012458496, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.25.attn_v.bias", "offset": 47146676224, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 47146680320, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.25.attn_norm.weight", "offset": 47163457536, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 47163490304, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.25.ffn_up.weight", "offset": 47647932416, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.25.ffn_norm.weight", "offset": 48132374528, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 48132407296, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 48132440064, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.26.ffn_gate.weight", "offset": 48616882176, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.26.ffn_up.weight", "offset": 49101324288, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.26.ffn_norm.weight", "offset": 49585766400, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 49585799168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 49585803264, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.26.attn_output.weight", "offset": 49602580480, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.26.attn_q.bias", "offset": 49736798208, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 49736830976, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.26.attn_v.bias", "offset": 49871048704, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 49871052800, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.27.ffn_gate.weight", "offset": 49887830016, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.27.ffn_up.weight", "offset": 50372272128, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.27.attn_k.bias", "offset": 50856714240, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 50856718336, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.27.attn_output.weight", "offset": 50873495552, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.27.attn_q.bias", "offset": 51007713280, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 51007746048, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.27.attn_v.bias", "offset": 51141963776, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 51141967872, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.27.attn_norm.weight", "offset": 51158745088, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 51158777856, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.27.ffn_norm.weight", "offset": 51643219968, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 51643252736, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 51643285504, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.28.ffn_gate.weight", "offset": 52127727616, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.28.ffn_up.weight", "offset": 52612169728, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.28.ffn_norm.weight", "offset": 53096611840, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 53096644608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 53096648704, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.28.attn_output.weight", "offset": 53113425920, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.28.attn_q.bias", "offset": 53247643648, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 53247676416, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "\u0001\u0000\u0000\u0000\u0000\u0004\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0010", "offset": 53381894144, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 53381898240, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.29.attn_norm.weight", "offset": 53398675456, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 53398708224, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.29.ffn_gate.weight", "offset": 53883150336, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.29.ffn_up.weight", "offset": 54367592448, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.29.ffn_norm.weight", "offset": 54852034560, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 54852067328, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 54852071424, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.29.attn_output.weight", "offset": 54868848640, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.29.attn_q.bias", "offset": 55003066368, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 55003099136, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.29.attn_v.bias", "offset": 55137316864, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 55137320960, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.30.attn_norm.weight", "offset": 55154098176, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 55154130944, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.30.ffn_gate.weight", "offset": 55638573056, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.30.ffn_up.weight", "offset": 56123015168, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.30.ffn_norm.weight", "offset": 56607457280, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 56607490048, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 56607494144, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.30.attn_output.weight", "offset": 56624271360, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.30.attn_q.bias", "offset": 56758489088, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 56758521856, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.30.attn_v.bias", "offset": 56892739584, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 56892743680, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.31.attn_norm.weight", "offset": 56909520896, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 56909553664, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.31.ffn_gate.weight", "offset": 57393995776, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.31.ffn_up.weight", "offset": 57878437888, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.31.ffn_norm.weight", "offset": 58362880000, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 58362912768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 58362916864, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.31.attn_output.weight", "offset": 58379694080, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.31.attn_q.bias", "offset": 58513911808, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 58513944576, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.31.attn_v.bias", "offset": 58648162304, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 58648166400, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.32.attn_k.bias", "offset": 58664943616, "shape": [ 1024 ], "size": 16777216, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 58664947712, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.32.attn_output.weight", "offset": 58681724928, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.32.attn_q.bias", "offset": 58815942656, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 58815975424, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.32.attn_v.bias", "offset": 58950193152, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 58950197248, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.32.attn_norm.weight", "offset": 58966974464, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 58967007232, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.32.ffn_gate.weight", "offset": 59451449344, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.32.ffn_up.weight", "offset": 59935891456, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.32.ffn_norm.weight", "offset": 60420333568, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 60420366336, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 60420399104, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.33.ffn_gate.weight", "offset": 60904841216, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.33.ffn_up.weight", "offset": 61389283328, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.33.ffn_norm.weight", "offset": 61873725440, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 61873758208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 61873762304, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.33.attn_output.weight", "offset": 61890539520, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.33.attn_q.bias", "offset": 62024757248, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 62024790016, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.33.attn_v.bias", "offset": 62159007744, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 62159011840, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.34.ffn_gate.weight", "offset": 62175789056, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.34.attn_k.bias", "offset": 62660231168, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 62660235264, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.34.attn_output.weight", "offset": 62677012480, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.34.attn_q.bias", "offset": 62811230208, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 62811262976, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.34.attn_v.bias", "offset": 62945480704, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 62945484800, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.34.attn_norm.weight", "offset": 62962262016, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 62962294784, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.34.ffn_up.weight", "offset": 63446736896, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.34.ffn_norm.weight", "offset": 63931179008, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 63931211776, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 63931244544, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.35.ffn_gate.weight", "offset": 64415686656, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.35.ffn_up.weight", "offset": 64900128768, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.35.ffn_norm.weight", "offset": 65384570880, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 65384603648, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 65384607744, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.35.attn_output.weight", "offset": 65401384960, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.35.attn_q.bias", "offset": 65535602688, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 65535635456, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.35.attn_v.bias", "offset": 65669853184, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 65669857280, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.36.ffn_gate.weight", "offset": 65686634496, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.36.ffn_up.weight", "offset": 66171076608, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.36.attn_k.bias", "offset": 66655518720, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 66655522816, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.36.attn_output.weight", "offset": 66672300032, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.36.attn_q.bias", "offset": 66806517760, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 66806550528, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.36.attn_v.bias", "offset": 66940768256, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 66940772352, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.36.attn_norm.weight", "offset": 66957549568, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 66957582336, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.36.ffn_norm.weight", "offset": 67442024448, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 67442057216, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 67442089984, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.37.ffn_gate.weight", "offset": 67926532096, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.37.ffn_up.weight", "offset": 68410974208, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.37.ffn_norm.weight", "offset": 68895416320, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 68895449088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 68895453184, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.37.attn_output.weight", "offset": 68912230400, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.37.attn_q.bias", "offset": 69046448128, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 69046480896, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.37.attn_v.bias", "offset": 69180698624, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 69180702720, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.38.attn_norm.weight", "offset": 69197479936, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 69197512704, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.38.ffn_gate.weight", "offset": 69681954816, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.38.ffn_up.weight", "offset": 70166396928, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.38.ffn_norm.weight", "offset": 70650839040, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 70650871808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 70650875904, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.38.attn_output.weight", "offset": 70667653120, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.38.attn_q.bias", "offset": 70801870848, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 70801903616, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.38.attn_v.bias", "offset": 70936121344, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 70936125440, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.39.attn_norm.weight", "offset": 70952902656, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 70952935424, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.39.ffn_gate.weight", "offset": 71437377536, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.39.ffn_up.weight", "offset": 71921819648, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.39.ffn_norm.weight", "offset": 72406261760, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 72406294528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 72406298624, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.39.attn_output.weight", "offset": 72423075840, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.39.attn_q.bias", "offset": 72557293568, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 72557326336, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.39.attn_v.bias", "offset": 72691544064, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 72691548160, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.40.attn_norm.weight", "offset": 72708325376, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 72708358144, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.40.ffn_gate.weight", "offset": 73192800256, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.40.ffn_up.weight", "offset": 73677242368, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.40.ffn_norm.weight", "offset": 74161684480, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 74161717248, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 74161721344, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.40.attn_output.weight", "offset": 74178498560, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.40.attn_q.bias", "offset": 74312716288, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 74312749056, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.40.attn_v.bias", "offset": 74446966784, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 74446970880, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.41.attn_k.bias", "offset": 74463748096, "shape": [ 1024 ], "size": 16777216, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 74463752192, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.41.attn_output.weight", "offset": 74480529408, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.41.attn_q.bias", "offset": 74614747136, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 74614779904, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.41.attn_v.bias", "offset": 74748997632, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 74749001728, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.41.attn_norm.weight", "offset": 74765778944, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 74765811712, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.41.ffn_gate.weight", "offset": 75250253824, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.41.ffn_up.weight", "offset": 75734695936, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.41.ffn_norm.weight", "offset": 76219138048, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 76219170816, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 76219203584, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.42.ffn_gate.weight", "offset": 76703645696, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.42.ffn_up.weight", "offset": 77188087808, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.42.ffn_norm.weight", "offset": 77672529920, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 77672562688, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 77672566784, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.42.attn_output.weight", "offset": 77689344000, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.42.attn_q.bias", "offset": 77823561728, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 77823594496, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.42.attn_v.bias", "offset": 77957812224, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 77957816320, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.43.ffn_gate.weight", "offset": 77974593536, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.43.attn_k.bias", "offset": 78459035648, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 78459039744, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.43.attn_output.weight", "offset": 78475816960, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.43.attn_q.bias", "offset": 78610034688, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 78610067456, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.43.attn_v.bias", "offset": 78744285184, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 78744289280, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.43.attn_norm.weight", "offset": 78761066496, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 78761099264, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.43.ffn_up.weight", "offset": 79245541376, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.43.ffn_norm.weight", "offset": 79729983488, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 79730016256, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 79730049024, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.44.ffn_gate.weight", "offset": 80214491136, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.44.ffn_up.weight", "offset": 80698933248, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.44.ffn_norm.weight", "offset": 81183375360, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 81183408128, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 81183412224, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.44.attn_output.weight", "offset": 81200189440, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.44.attn_q.bias", "offset": 81334407168, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 81334439936, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.44.attn_v.bias", "offset": 81468657664, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 81468661760, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.45.ffn_gate.weight", "offset": 81485438976, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.45.ffn_up.weight", "offset": 81969881088, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.45.attn_k.bias", "offset": 82454323200, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 82454327296, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.45.attn_output.weight", "offset": 82471104512, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.45.attn_q.bias", "offset": 82605322240, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 82605355008, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.45.attn_v.bias", "offset": 82739572736, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 82739576832, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.45.attn_norm.weight", "offset": 82756354048, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 82756386816, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.45.ffn_norm.weight", "offset": 83240828928, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 83240861696, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 83240894464, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.46.ffn_gate.weight", "offset": 83725336576, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.46.ffn_up.weight", "offset": 84209778688, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.46.ffn_norm.weight", "offset": 84694220800, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 84694253568, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 84694257664, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.46.attn_output.weight", "offset": 84711034880, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.46.attn_q.bias", "offset": 84845252608, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 84845285376, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.46.attn_v.bias", "offset": 84979503104, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 84979507200, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.47.attn_norm.weight", "offset": 84996284416, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 84996317184, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.47.ffn_gate.weight", "offset": 85480759296, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.47.ffn_up.weight", "offset": 85965201408, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.47.ffn_norm.weight", "offset": 86449643520, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 86449676288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 86449680384, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.47.attn_output.weight", "offset": 86466457600, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.47.attn_q.bias", "offset": 86600675328, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 86600708096, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.47.attn_v.bias", "offset": 86734925824, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 86734929920, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.48.attn_norm.weight", "offset": 86751707136, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 86751739904, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.48.ffn_gate.weight", "offset": 87236182016, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.48.ffn_up.weight", "offset": 87720624128, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.48.ffn_norm.weight", "offset": 88205066240, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 88205099008, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 88205103104, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.48.attn_output.weight", "offset": 88221880320, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.48.attn_q.bias", "offset": 88356098048, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 88356130816, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.48.attn_v.bias", "offset": 88490348544, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 88490352640, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.49.attn_norm.weight", "offset": 88507129856, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 88507162624, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.49.ffn_gate.weight", "offset": 88991604736, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.49.ffn_up.weight", "offset": 89476046848, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.49.ffn_norm.weight", "offset": 89960488960, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 89960521728, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 89960525824, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.49.attn_output.weight", "offset": 89977303040, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.49.attn_q.bias", "offset": 90111520768, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 90111553536, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.49.attn_v.bias", "offset": 90245771264, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 90245775360, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.50.attn_k.bias", "offset": 90262552576, "shape": [ 1024 ], "size": 16777216, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 90262556672, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.50.attn_output.weight", "offset": 90279333888, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.50.attn_q.bias", "offset": 90413551616, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 90413584384, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.50.attn_v.bias", "offset": 90547802112, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 90547806208, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.50.attn_norm.weight", "offset": 90564583424, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 90564616192, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.50.ffn_gate.weight", "offset": 91049058304, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.50.ffn_up.weight", "offset": 91533500416, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.50.ffn_norm.weight", "offset": 92017942528, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 92017975296, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 92018008064, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.51.ffn_gate.weight", "offset": 92502450176, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.51.ffn_up.weight", "offset": 92986892288, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.51.ffn_norm.weight", "offset": 93471334400, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 93471367168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 93471371264, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.51.attn_output.weight", "offset": 93488148480, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.51.attn_q.bias", "offset": 93622366208, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 93622398976, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.51.attn_v.bias", "offset": 93756616704, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 93756620800, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.52.ffn_gate.weight", "offset": 93773398016, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.52.attn_k.bias", "offset": 94257840128, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 94257844224, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.52.attn_output.weight", "offset": 94274621440, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.52.attn_q.bias", "offset": 94408839168, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 94408871936, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.52.attn_v.bias", "offset": 94543089664, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 94543093760, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.52.attn_norm.weight", "offset": 94559870976, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 94559903744, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.52.ffn_up.weight", "offset": 95044345856, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.52.ffn_norm.weight", "offset": 95528787968, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 95528820736, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 95528853504, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.53.ffn_gate.weight", "offset": 96013295616, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.53.ffn_up.weight", "offset": 96497737728, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.53.ffn_norm.weight", "offset": 96982179840, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 96982212608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 96982216704, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.53.attn_output.weight", "offset": 96998993920, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.53.attn_q.bias", "offset": 97133211648, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 97133244416, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.53.attn_v.bias", "offset": 97267462144, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 97267466240, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.54.ffn_gate.weight", "offset": 97284243456, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.54.ffn_up.weight", "offset": 97768685568, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.54.attn_k.bias", "offset": 98253127680, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 98253131776, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.54.attn_output.weight", "offset": 98269908992, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.54.attn_q.bias", "offset": 98404126720, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 98404159488, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.54.attn_v.bias", "offset": 98538377216, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 98538381312, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.54.attn_norm.weight", "offset": 98555158528, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 98555191296, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.54.ffn_norm.weight", "offset": 99039633408, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 99039666176, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 99039698944, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.55.ffn_gate.weight", "offset": 99524141056, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.55.ffn_up.weight", "offset": 100008583168, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.55.ffn_norm.weight", "offset": 100493025280, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 100493058048, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 100493062144, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.55.attn_output.weight", "offset": 100509839360, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.55.attn_q.bias", "offset": 100644057088, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 100644089856, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.55.attn_v.bias", "offset": 100778307584, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 100778311680, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.56.attn_norm.weight", "offset": 100795088896, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 100795121664, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.56.ffn_gate.weight", "offset": 101279563776, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.56.ffn_up.weight", "offset": 101764005888, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.56.ffn_norm.weight", "offset": 102248448000, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 102248480768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 102248484864, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.56.attn_output.weight", "offset": 102265262080, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.56.attn_q.bias", "offset": 102399479808, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 102399512576, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.56.attn_v.bias", "offset": 102533730304, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 102533734400, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.57.attn_norm.weight", "offset": 102550511616, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 102550544384, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.57.ffn_gate.weight", "offset": 103034986496, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.57.ffn_up.weight", "offset": 103519428608, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.57.ffn_norm.weight", "offset": 104003870720, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 104003903488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 104003907584, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.57.attn_output.weight", "offset": 104020684800, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.57.attn_q.bias", "offset": 104154902528, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 104154935296, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.57.attn_v.bias", "offset": 104289153024, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 104289157120, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.58.attn_norm.weight", "offset": 104305934336, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 104305967104, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.58.ffn_gate.weight", "offset": 104790409216, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.58.ffn_up.weight", "offset": 105274851328, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.58.ffn_norm.weight", "offset": 105759293440, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 105759326208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 105759330304, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.58.attn_output.weight", "offset": 105776107520, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.58.attn_q.bias", "offset": 105910325248, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 105910358016, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.58.attn_v.bias", "offset": 106044575744, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 106044579840, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.59.attn_k.bias", "offset": 106061357056, "shape": [ 1024 ], "size": 16777216, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 106061361152, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.59.attn_output.weight", "offset": 106078138368, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.59.attn_q.bias", "offset": 106212356096, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 106212388864, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.59.attn_v.bias", "offset": 106346606592, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 106346610688, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.59.attn_norm.weight", "offset": 106363387904, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 106363420672, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.59.ffn_gate.weight", "offset": 106847862784, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.59.ffn_up.weight", "offset": 107332304896, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.59.ffn_norm.weight", "offset": 107816747008, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 107816779776, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 107816812544, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.60.ffn_gate.weight", "offset": 108301254656, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.60.ffn_up.weight", "offset": 108785696768, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.60.ffn_norm.weight", "offset": 109270138880, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 109270171648, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 109270175744, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.60.attn_output.weight", "offset": 109286952960, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.60.attn_q.bias", "offset": 109421170688, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 109421203456, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.60.attn_v.bias", "offset": 109555421184, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 109555425280, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.61.ffn_gate.weight", "offset": 109572202496, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.61.attn_k.bias", "offset": 110056644608, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 110056648704, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.61.attn_output.weight", "offset": 110073425920, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.61.attn_q.bias", "offset": 110207643648, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 110207676416, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.61.attn_v.bias", "offset": 110341894144, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 110341898240, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.61.attn_norm.weight", "offset": 110358675456, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 110358708224, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.61.ffn_up.weight", "offset": 110843150336, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.61.ffn_norm.weight", "offset": 111327592448, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 111327625216, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 111327657984, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.62.ffn_gate.weight", "offset": 111812100096, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.62.ffn_up.weight", "offset": 112296542208, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.62.ffn_norm.weight", "offset": 112780984320, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 112781017088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 112781021184, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.62.attn_output.weight", "offset": 112797798400, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.62.attn_q.bias", "offset": 112932016128, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 112932048896, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.62.attn_v.bias", "offset": 113066266624, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 113066270720, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.63.ffn_gate.weight", "offset": 113083047936, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.63.ffn_up.weight", "offset": 113567490048, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.63.attn_k.bias", "offset": 114051932160, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 114051936256, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.63.attn_output.weight", "offset": 114068713472, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.63.attn_q.bias", "offset": 114202931200, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 114202963968, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.63.attn_v.bias", "offset": 114337181696, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 114337185792, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.63.attn_norm.weight", "offset": 114353963008, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 114353995776, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.63.ffn_norm.weight", "offset": 114838437888, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 114838470656, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 114838503424, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.64.ffn_gate.weight", "offset": 115322945536, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.64.ffn_up.weight", "offset": 115807387648, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.64.ffn_norm.weight", "offset": 116291829760, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 116291862528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "\u0000\u0000\u001c\ufffd\u001f\u0000\u0000\u0000\u0012\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.", "offset": 116291866624, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.64.attn_output.weight", "offset": 116308643840, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.64.attn_q.bias", "offset": 116442861568, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 116442894336, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.64.attn_v.bias", "offset": 116577112064, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 116577116160, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.65.attn_norm.weight", "offset": 116593893376, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 116593926144, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.65.ffn_gate.weight", "offset": 117078368256, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.65.ffn_up.weight", "offset": 117562810368, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.65.ffn_norm.weight", "offset": 118047252480, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 118047285248, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 118047289344, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.65.attn_output.weight", "offset": 118064066560, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.65.attn_q.bias", "offset": 118198284288, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 118198317056, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.65.attn_v.bias", "offset": 118332534784, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 118332538880, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.66.attn_norm.weight", "offset": 118349316096, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 118349348864, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.66.ffn_gate.weight", "offset": 118833790976, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.66.ffn_up.weight", "offset": 119318233088, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.66.ffn_norm.weight", "offset": 119802675200, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 119802707968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 119802712064, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.66.attn_output.weight", "offset": 119819489280, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.66.attn_q.bias", "offset": 119953707008, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 119953739776, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.66.attn_v.bias", "offset": 120087957504, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 120087961600, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.67.attn_norm.weight", "offset": 120104738816, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 120104771584, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.67.ffn_gate.weight", "offset": 120589213696, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.67.ffn_up.weight", "offset": 121073655808, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.67.ffn_norm.weight", "offset": 121558097920, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 121558130688, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 121558134784, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.67.attn_output.weight", "offset": 121574912000, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.67.attn_q.bias", "offset": 121709129728, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 121709162496, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.67.attn_v.bias", "offset": 121843380224, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 121843384320, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.68.attn_k.bias", "offset": 121860161536, "shape": [ 1024 ], "size": 16777216, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 121860165632, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.68.attn_output.weight", "offset": 121876942848, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.68.attn_q.bias", "offset": 122011160576, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 122011193344, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.68.attn_v.bias", "offset": 122145411072, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 122145415168, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.68.attn_norm.weight", "offset": 122162192384, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 122162225152, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.68.ffn_gate.weight", "offset": 122646667264, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.68.ffn_up.weight", "offset": 123131109376, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.68.ffn_norm.weight", "offset": 123615551488, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 123615584256, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 123615617024, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.69.ffn_gate.weight", "offset": 124100059136, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.69.ffn_up.weight", "offset": 124584501248, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.69.ffn_norm.weight", "offset": 125068943360, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 125068976128, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 125068980224, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.69.attn_output.weight", "offset": 125085757440, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.69.attn_q.bias", "offset": 125219975168, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 125220007936, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.69.attn_v.bias", "offset": 125354225664, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 125354229760, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.70.ffn_gate.weight", "offset": 125371006976, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.70.attn_k.bias", "offset": 125855449088, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 125855453184, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.70.attn_output.weight", "offset": 125872230400, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.70.attn_q.bias", "offset": 126006448128, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 126006480896, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.70.attn_v.bias", "offset": 126140698624, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 126140702720, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.70.attn_norm.weight", "offset": 126157479936, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 126157512704, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.70.ffn_up.weight", "offset": 126641954816, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.70.ffn_norm.weight", "offset": 127126396928, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 127126429696, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 127126462464, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.71.ffn_gate.weight", "offset": 127610904576, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.71.ffn_up.weight", "offset": 128095346688, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.71.ffn_norm.weight", "offset": 128579788800, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 128579821568, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 128579825664, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.71.attn_output.weight", "offset": 128596602880, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.71.attn_q.bias", "offset": 128730820608, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 128730853376, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.71.attn_v.bias", "offset": 128865071104, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 128865075200, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.72.ffn_gate.weight", "offset": 128881852416, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.72.ffn_up.weight", "offset": 129366294528, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.72.attn_k.bias", "offset": 129850736640, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 129850740736, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.72.attn_output.weight", "offset": 129867517952, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.72.attn_q.bias", "offset": 130001735680, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 130001768448, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.72.attn_v.bias", "offset": 130135986176, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 130135990272, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.72.attn_norm.weight", "offset": 130152767488, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 130152800256, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.72.ffn_norm.weight", "offset": 130637242368, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 130637275136, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 130637307904, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.73.ffn_gate.weight", "offset": 131121750016, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.73.ffn_up.weight", "offset": 131606192128, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.73.ffn_norm.weight", "offset": 132090634240, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 132090667008, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 132090671104, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.73.attn_output.weight", "offset": 132107448320, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.73.attn_q.bias", "offset": 132241666048, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 132241698816, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.73.attn_v.bias", "offset": 132375916544, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 132375920640, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.74.attn_norm.weight", "offset": 132392697856, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 132392730624, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.74.ffn_gate.weight", "offset": 132877172736, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.74.ffn_up.weight", "offset": 133361614848, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.74.ffn_norm.weight", "offset": 133846056960, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 133846089728, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 133846093824, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.74.attn_output.weight", "offset": 133862871040, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.74.attn_q.bias", "offset": 133997088768, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 133997121536, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.74.attn_v.bias", "offset": 134131339264, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 134131343360, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.75.attn_norm.weight", "offset": 134148120576, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 134148153344, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.75.ffn_gate.weight", "offset": 134632595456, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.75.ffn_up.weight", "offset": 135117037568, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.75.ffn_norm.weight", "offset": 135601479680, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 135601512448, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 135601516544, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.75.attn_output.weight", "offset": 135618293760, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.75.attn_q.bias", "offset": 135752511488, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 135752544256, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.75.attn_v.bias", "offset": 135886761984, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 135886766080, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.76.attn_norm.weight", "offset": 135903543296, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 135903576064, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.76.ffn_gate.weight", "offset": 136388018176, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.76.ffn_up.weight", "offset": 136872460288, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.76.ffn_norm.weight", "offset": 137356902400, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 137356935168, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 137356939264, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.76.attn_output.weight", "offset": 137373716480, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.76.attn_q.bias", "offset": 137507934208, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 137507966976, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.76.attn_v.bias", "offset": 137642184704, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 137642188800, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.77.attn_k.bias", "offset": 137658966016, "shape": [ 1024 ], "size": 16777216, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 137658970112, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.77.attn_output.weight", "offset": 137675747328, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.77.attn_q.bias", "offset": 137809965056, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 137809997824, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.77.attn_v.bias", "offset": 137944215552, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 137944219648, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.77.attn_norm.weight", "offset": 137960996864, "shape": [ 8192 ], "size": 16777216, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 137961029632, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.77.ffn_gate.weight", "offset": 138445471744, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.77.ffn_up.weight", "offset": 138929913856, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.77.ffn_norm.weight", "offset": 139414355968, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 139414388736, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 139414421504, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.78.ffn_gate.weight", "offset": 139898863616, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.78.ffn_up.weight", "offset": 140383305728, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.78.ffn_norm.weight", "offset": 140867747840, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 140867780608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 140867784704, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.78.attn_output.weight", "offset": 140884561920, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.78.attn_q.bias", "offset": 141018779648, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 141018812416, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.78.attn_v.bias", "offset": 141153030144, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 141153034240, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.79.ffn_gate.weight", "offset": 141169811456, "shape": [ 8192, 29568 ], "size": 16777216, "type": 1 }, { "name": "blk.79.attn_k.bias", "offset": 141654253568, "shape": [ 1024 ], "size": 484442112, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 141654257664, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "blk.79.attn_output.weight", "offset": 141671034880, "shape": [ 8192, 8192 ], "size": 16777216, "type": 1 }, { "name": "blk.79.attn_q.bias", "offset": 141805252608, "shape": [ 8192 ], "size": 134217728, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 141805285376, "shape": [ 8192, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.79.attn_v.bias", "offset": 141939503104, "shape": [ 1024 ], "size": 134217728, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 141939507200, "shape": [ 8192, 1024 ], "size": 4096, "type": 1 }, { "name": "output.weight", "offset": 141956284416, "shape": [ 8192, 152064 ], "size": 16777216, "type": 1 }, { "name": "blk.79.attn_norm.weight", "offset": 144447700992, "shape": [ 8192 ], "size": 2491416576, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 144447733760, "shape": [ 29568, 8192 ], "size": 32768, "type": 1 }, { "name": "blk.79.ffn_up.weight", "offset": 144932175872, "shape": [ 8192, 29568 ], "size": 484442112, "type": 1 }, { "name": "blk.79.ffn_norm.weight", "offset": 145416617984, "shape": [ 8192 ], "size": 484442112, "type": 0 }, { "name": "output_norm.weight", "offset": 145416650752, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }