qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 2, "general.name": "Qwen2-72B", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.00001, "qwen2.block_count": 80, "qwen2.context_length": 131072, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 700710912, "shape": [ 8192, 29568 ], "size": 700710912, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 836960256, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.0.attn_k.bias", "offset": 973209600, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 973213696, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.0.attn_output.weight", "offset": 977932288, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.0.attn_q.bias", "offset": 1015681024, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 1015713792, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.0.attn_v.bias", "offset": 1053462528, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 1053466624, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.0.attn_norm.weight", "offset": 1058185216, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 1058217984, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 1194467328, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1194500096, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1194532864, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.1.ffn_gate.weight", "offset": 1330782208, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 1467031552, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 1603280896, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 1603313664, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1603317760, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.1.attn_output.weight", "offset": 1608036352, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.1.attn_q.bias", "offset": 1645785088, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 1645817856, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.1.attn_v.bias", "offset": 1683566592, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 1683570688, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 1688289280, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1688322048, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.2.ffn_gate.weight", "offset": 1824571392, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 1960820736, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 2097070080, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 2097102848, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 2097106944, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.2.attn_output.weight", "offset": 2101825536, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.2.attn_q.bias", "offset": 2139574272, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 2139607040, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.2.attn_v.bias", "offset": 2177355776, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 2177359872, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 2182078464, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 2182111232, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.3.ffn_gate.weight", "offset": 2318360576, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 2454609920, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 2590859264, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 2590892032, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 2590896128, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.3.attn_output.weight", "offset": 2595614720, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.3.attn_q.bias", "offset": 2633363456, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 2633396224, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.3.attn_v.bias", "offset": 2671144960, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 2671149056, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 2675867648, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 2675900416, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.4.ffn_gate.weight", "offset": 2812149760, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 2948399104, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 3084648448, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 3084681216, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 3084685312, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.4.attn_output.weight", "offset": 3089403904, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.4.attn_q.bias", "offset": 3127152640, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 3127185408, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.4.attn_v.bias", "offset": 3164934144, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 3164938240, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.5.attn_k.bias", "offset": 3169656832, "shape": [ 1024 ], "size": 4718592, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 3169660928, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.5.attn_output.weight", "offset": 3174379520, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.5.attn_q.bias", "offset": 3212128256, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 3212161024, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.5.attn_v.bias", "offset": 3249909760, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 3249913856, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 3254632448, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 3254665216, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.5.ffn_gate.weight", "offset": 3390914560, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 3527163904, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 3663413248, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 3663446016, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 3663478784, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.6.ffn_gate.weight", "offset": 3799728128, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 3935977472, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 4072226816, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 4072259584, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 4072263680, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 4076982272, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.6.attn_q.bias", "offset": 4114731008, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 4114763776, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.6.attn_v.bias", "offset": 4152512512, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 4152516608, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.7.ffn_gate.weight", "offset": 4157235200, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.7.attn_k.bias", "offset": 4293484544, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 4293488640, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.7.attn_output.weight", "offset": 4298207232, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.7.attn_q.bias", "offset": 4335955968, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 4335988736, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.7.attn_v.bias", "offset": 4373737472, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 4373741568, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.7.attn_norm.weight", "offset": 4378460160, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 4378492928, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 4514742272, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 4650991616, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 4651024384, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 4651057152, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.8.ffn_gate.weight", "offset": 4787306496, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 4923555840, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 5059805184, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 5059837952, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 5059842048, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.8.attn_output.weight", "offset": 5064560640, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.8.attn_q.bias", "offset": 5102309376, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 5102342144, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.8.attn_v.bias", "offset": 5140090880, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 5140094976, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.9.ffn_gate.weight", "offset": 5144813568, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 5281062912, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.9.attn_k.bias", "offset": 5417312256, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 5417316352, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.9.attn_output.weight", "offset": 5422034944, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.9.attn_q.bias", "offset": 5459783680, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 5459816448, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.9.attn_v.bias", "offset": 5497565184, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 5497569280, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 5502287872, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 5502320640, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.10.ffn_gate.weight", "offset": 5638569984, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 5774819328, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 5911068672, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 5911101440, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 5911105536, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.10.attn_output.weight", "offset": 5915824128, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.10.attn_q.bias", "offset": 5953572864, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 5953605632, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.10.attn_v.bias", "offset": 5991354368, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 5991358464, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 5996077056, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 5996109824, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.11.ffn_gate.weight", "offset": 6132359168, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 6268608512, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 6404857856, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 6404890624, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 6404894720, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.11.attn_output.weight", "offset": 6409613312, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.11.attn_q.bias", "offset": 6447362048, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 6447394816, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.11.attn_v.bias", "offset": 6485143552, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 6485147648, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 6489866240, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 6489899008, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 6626148352, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 6626181120, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 6626213888, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.12.ffn_gate.weight", "offset": 6762463232, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 6898712576, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 7034961920, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 7034994688, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 7034998784, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.12.attn_output.weight", "offset": 7039717376, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.12.attn_q.bias", "offset": 7077466112, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 7077498880, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.12.attn_v.bias", "offset": 7115247616, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 7115251712, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 7119970304, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 7120003072, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.13.ffn_gate.weight", "offset": 7256252416, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 7392501760, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 7528751104, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 7528783872, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 7528787968, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.13.attn_output.weight", "offset": 7533506560, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.13.attn_q.bias", "offset": 7571255296, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 7571288064, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.13.attn_v.bias", "offset": 7609036800, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 7609040896, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.14.attn_k.bias", "offset": 7613759488, "shape": [ 1024 ], "size": 4718592, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 7613763584, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.14.attn_output.weight", "offset": 7618482176, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.14.attn_q.bias", "offset": 7656230912, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 7656263680, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.14.attn_v.bias", "offset": 7694012416, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 7694016512, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 7698735104, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 7698767872, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.14.ffn_gate.weight", "offset": 7835017216, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 7971266560, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 8107515904, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 8107548672, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 8107581440, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.15.ffn_gate.weight", "offset": 8243830784, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 8380080128, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 8516329472, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 8516362240, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 8516366336, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.15.attn_output.weight", "offset": 8521084928, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.15.attn_q.bias", "offset": 8558833664, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 8558866432, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.15.attn_v.bias", "offset": 8596615168, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 8596619264, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.16.ffn_gate.weight", "offset": 8601337856, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.16.attn_k.bias", "offset": 8737587200, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 8737591296, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.16.attn_output.weight", "offset": 8742309888, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.16.attn_q.bias", "offset": 8780058624, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 8780091392, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.16.attn_v.bias", "offset": 8817840128, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 8817844224, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 8822562816, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 8822595584, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 8958844928, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 9095094272, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 9095127040, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 9095159808, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.17.ffn_gate.weight", "offset": 9231409152, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 9367658496, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 9503907840, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 9503940608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 9503944704, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.17.attn_output.weight", "offset": 9508663296, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.17.attn_q.bias", "offset": 9546412032, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 9546444800, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.17.attn_v.bias", "offset": 9584193536, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 9584197632, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.18.ffn_gate.weight", "offset": 9588916224, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 9725165568, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.18.attn_k.bias", "offset": 9861414912, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 9861419008, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.18.attn_output.weight", "offset": 9866137600, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.18.attn_q.bias", "offset": 9903886336, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 9903919104, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.18.attn_v.bias", "offset": 9941667840, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 9941671936, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 9946390528, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 9946423296, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 10082672640, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 10082705408, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 10082738176, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.19.ffn_gate.weight", "offset": 10218987520, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 10355236864, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.19.ffn_norm.weight", "offset": 10491486208, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 10491518976, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 10491523072, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.19.attn_output.weight", "offset": 10496241664, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.19.attn_q.bias", "offset": 10533990400, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 10534023168, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.19.attn_v.bias", "offset": 10571771904, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 10571776000, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 10576494592, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 10576527360, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.20.ffn_gate.weight", "offset": 10712776704, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 10849026048, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 10985275392, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 10985308160, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 10985312256, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.20.attn_output.weight", "offset": 10990030848, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.20.attn_q.bias", "offset": 11027779584, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 11027812352, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.20.attn_v.bias", "offset": 11065561088, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 11065565184, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 11070283776, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 11070316544, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.21.ffn_gate.weight", "offset": 11206565888, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 11342815232, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 11479064576, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 11479097344, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 11479101440, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 11483820032, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.21.attn_q.bias", "offset": 11521568768, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 11521601536, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.21.attn_v.bias", "offset": 11559350272, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 11559354368, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.22.attn_norm.weight", "offset": 11564072960, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 11564105728, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.22.ffn_gate.weight", "offset": 11700355072, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.22.ffn_up.weight", "offset": 11836604416, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 11972853760, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 11972886528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 11972890624, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.22.attn_output.weight", "offset": 11977609216, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.22.attn_q.bias", "offset": 12015357952, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 12015390720, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.22.attn_v.bias", "offset": 12053139456, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 12053143552, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.23.attn_k.bias", "offset": 12057862144, "shape": [ 1024 ], "size": 4718592, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 12057866240, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.23.attn_output.weight", "offset": 12062584832, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.23.attn_q.bias", "offset": 12100333568, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 12100366336, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.23.attn_v.bias", "offset": 12138115072, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 12138119168, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.23.attn_norm.weight", "offset": 12142837760, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 12142870528, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.23.ffn_gate.weight", "offset": 12279119872, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 12415369216, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 12551618560, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 12551651328, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 12551684096, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.24.ffn_gate.weight", "offset": 12687933440, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.24.ffn_up.weight", "offset": 12824182784, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.24.ffn_norm.weight", "offset": 12960432128, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 12960464896, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 12960468992, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.24.attn_output.weight", "offset": 12965187584, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.24.attn_q.bias", "offset": 13002936320, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 13002969088, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.24.attn_v.bias", "offset": 13040717824, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 13040721920, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.25.ffn_gate.weight", "offset": 13045440512, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.25.attn_k.bias", "offset": 13181689856, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 13181693952, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.25.attn_output.weight", "offset": 13186412544, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.25.attn_q.bias", "offset": 13224161280, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 13224194048, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.25.attn_v.bias", "offset": 13261942784, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 13261946880, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.25.attn_norm.weight", "offset": 13266665472, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 13266698240, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.25.ffn_up.weight", "offset": 13402947584, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.25.ffn_norm.weight", "offset": 13539196928, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 13539229696, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 13539262464, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.26.ffn_gate.weight", "offset": 13675511808, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.26.ffn_up.weight", "offset": 13811761152, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.26.ffn_norm.weight", "offset": 13948010496, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 13948043264, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 13948047360, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.26.attn_output.weight", "offset": 13952765952, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.26.attn_q.bias", "offset": 13990514688, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 13990547456, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.26.attn_v.bias", "offset": 14028296192, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 14028300288, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.27.ffn_gate.weight", "offset": 14033018880, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.27.ffn_up.weight", "offset": 14169268224, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.27.attn_k.bias", "offset": 14305517568, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 14305521664, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.27.attn_output.weight", "offset": 14310240256, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.27.attn_q.bias", "offset": 14347988992, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 14348021760, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.27.attn_v.bias", "offset": 14385770496, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 14385774592, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 14390493184, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 14390525952, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.27.ffn_norm.weight", "offset": 14526775296, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 14526808064, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 14526840832, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.28.ffn_gate.weight", "offset": 14663090176, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.28.ffn_up.weight", "offset": 14799339520, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.28.ffn_norm.weight", "offset": 14935588864, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 14935621632, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 14935625728, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.28.attn_output.weight", "offset": 14940344320, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.28.attn_q.bias", "offset": 14978093056, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 14978125824, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.28.attn_v.bias", "offset": 15015874560, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 15015878656, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.29.attn_norm.weight", "offset": 15020597248, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 15020630016, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.29.ffn_gate.weight", "offset": 15156879360, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.29.ffn_up.weight", "offset": 15293128704, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.29.ffn_norm.weight", "offset": 15429378048, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 15429410816, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 15429414912, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.29.attn_output.weight", "offset": 15434133504, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.29.attn_q.bias", "offset": 15471882240, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 15471915008, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.29.attn_v.bias", "offset": 15509663744, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 15509667840, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.30.attn_norm.weight", "offset": 15514386432, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 15514419200, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.30.ffn_gate.weight", "offset": 15650668544, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.30.ffn_up.weight", "offset": 15786917888, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.30.ffn_norm.weight", "offset": 15923167232, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 15923200000, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 15923204096, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.30.attn_output.weight", "offset": 15927922688, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.30.attn_q.bias", "offset": 15965671424, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 15965704192, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.30.attn_v.bias", "offset": 16003452928, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 16003457024, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.31.attn_norm.weight", "offset": 16008175616, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 16008208384, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.31.ffn_gate.weight", "offset": 16144457728, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.31.ffn_up.weight", "offset": 16280707072, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.31.ffn_norm.weight", "offset": 16416956416, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 16416989184, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 16416993280, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.31.attn_output.weight", "offset": 16421711872, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.31.attn_q.bias", "offset": 16459460608, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 16459493376, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.31.attn_v.bias", "offset": 16497242112, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 16497246208, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.32.attn_k.bias", "offset": 16501964800, "shape": [ 1024 ], "size": 4718592, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 16501968896, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.32.attn_output.weight", "offset": 16506687488, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.32.attn_q.bias", "offset": 16544436224, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 16544468992, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.32.attn_v.bias", "offset": 16582217728, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 16582221824, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.32.attn_norm.weight", "offset": 16586940416, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 16586973184, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.32.ffn_gate.weight", "offset": 16723222528, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.32.ffn_up.weight", "offset": 16859471872, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.32.ffn_norm.weight", "offset": 16995721216, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 16995753984, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 16995786752, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.33.ffn_gate.weight", "offset": 17132036096, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.33.ffn_up.weight", "offset": 17268285440, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.33.ffn_norm.weight", "offset": 17404534784, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 17404567552, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 17404571648, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.33.attn_output.weight", "offset": 17409290240, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.33.attn_q.bias", "offset": 17447038976, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 17447071744, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.33.attn_v.bias", "offset": 17484820480, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 17484824576, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.34.ffn_gate.weight", "offset": 17489543168, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.34.attn_k.bias", "offset": 17625792512, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 17625796608, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.34.attn_output.weight", "offset": 17630515200, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.34.attn_q.bias", "offset": 17668263936, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 17668296704, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.34.attn_v.bias", "offset": 17706045440, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 17706049536, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.34.attn_norm.weight", "offset": 17710768128, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 17710800896, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.34.ffn_up.weight", "offset": 17847050240, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.34.ffn_norm.weight", "offset": 17983299584, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 17983332352, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 17983365120, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.35.ffn_gate.weight", "offset": 18119614464, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.35.ffn_up.weight", "offset": 18255863808, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.35.ffn_norm.weight", "offset": 18392113152, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 18392145920, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 18392150016, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.35.attn_output.weight", "offset": 18396868608, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.35.attn_q.bias", "offset": 18434617344, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 18434650112, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.35.attn_v.bias", "offset": 18472398848, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 18472402944, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.36.ffn_gate.weight", "offset": 18477121536, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.36.ffn_up.weight", "offset": 18613370880, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.36.attn_k.bias", "offset": 18749620224, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 18749624320, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.36.attn_output.weight", "offset": 18754342912, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.36.attn_q.bias", "offset": 18792091648, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 18792124416, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.36.attn_v.bias", "offset": 18829873152, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 18829877248, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.36.attn_norm.weight", "offset": 18834595840, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 18834628608, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.36.ffn_norm.weight", "offset": 18970877952, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 18970910720, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 18970943488, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.37.ffn_gate.weight", "offset": 19107192832, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.37.ffn_up.weight", "offset": 19243442176, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.37.ffn_norm.weight", "offset": 19379691520, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 19379724288, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 19379728384, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.37.attn_output.weight", "offset": 19384446976, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.37.attn_q.bias", "offset": 19422195712, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 19422228480, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.37.attn_v.bias", "offset": 19459977216, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 19459981312, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.38.attn_norm.weight", "offset": 19464699904, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 19464732672, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.38.ffn_gate.weight", "offset": 19600982016, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.38.ffn_up.weight", "offset": 19737231360, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.38.ffn_norm.weight", "offset": 19873480704, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 19873513472, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 19873517568, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.38.attn_output.weight", "offset": 19878236160, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.38.attn_q.bias", "offset": 19915984896, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 19916017664, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.38.attn_v.bias", "offset": 19953766400, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 19953770496, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.39.attn_norm.weight", "offset": 19958489088, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 19958521856, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.39.ffn_gate.weight", "offset": 20094771200, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.39.ffn_up.weight", "offset": 20231020544, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.39.ffn_norm.weight", "offset": 20367269888, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 20367302656, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 20367306752, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.39.attn_output.weight", "offset": 20372025344, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.39.attn_q.bias", "offset": 20409774080, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 20409806848, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.39.attn_v.bias", "offset": 20447555584, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 20447559680, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.40.attn_norm.weight", "offset": 20452278272, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 20452311040, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.40.ffn_gate.weight", "offset": 20588560384, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.40.ffn_up.weight", "offset": 20724809728, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.40.ffn_norm.weight", "offset": 20861059072, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 20861091840, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 20861095936, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.40.attn_output.weight", "offset": 20865814528, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.40.attn_q.bias", "offset": 20903563264, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 20903596032, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.40.attn_v.bias", "offset": 20941344768, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 20941348864, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.41.attn_k.bias", "offset": 20946067456, "shape": [ 1024 ], "size": 4718592, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 20946071552, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.41.attn_output.weight", "offset": 20950790144, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.41.attn_q.bias", "offset": 20988538880, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 20988571648, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.41.attn_v.bias", "offset": 21026320384, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 21026324480, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.41.attn_norm.weight", "offset": 21031043072, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 21031075840, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.41.ffn_gate.weight", "offset": 21167325184, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.41.ffn_up.weight", "offset": 21303574528, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.41.ffn_norm.weight", "offset": 21439823872, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 21439856640, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 21439889408, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.42.ffn_gate.weight", "offset": 21576138752, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.42.ffn_up.weight", "offset": 21712388096, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.42.ffn_norm.weight", "offset": 21848637440, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 21848670208, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 21848674304, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.42.attn_output.weight", "offset": 21853392896, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.42.attn_q.bias", "offset": 21891141632, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 21891174400, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.42.attn_v.bias", "offset": 21928923136, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 21928927232, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.43.ffn_gate.weight", "offset": 21933645824, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.43.attn_k.bias", "offset": 22069895168, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 22069899264, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.43.attn_output.weight", "offset": 22074617856, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.43.attn_q.bias", "offset": 22112366592, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 22112399360, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.43.attn_v.bias", "offset": 22150148096, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 22150152192, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.43.attn_norm.weight", "offset": 22154870784, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 22154903552, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.43.ffn_up.weight", "offset": 22291152896, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.43.ffn_norm.weight", "offset": 22427402240, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 22427435008, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 22427467776, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.44.ffn_gate.weight", "offset": 22563717120, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.44.ffn_up.weight", "offset": 22699966464, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.44.ffn_norm.weight", "offset": 22836215808, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 22836248576, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 22836252672, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.44.attn_output.weight", "offset": 22840971264, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.44.attn_q.bias", "offset": 22878720000, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 22878752768, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.44.attn_v.bias", "offset": 22916501504, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 22916505600, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.45.ffn_gate.weight", "offset": 22921224192, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.45.ffn_up.weight", "offset": 23057473536, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.45.attn_k.bias", "offset": 23193722880, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 23193726976, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.45.attn_output.weight", "offset": 23198445568, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.45.attn_q.bias", "offset": 23236194304, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 23236227072, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.45.attn_v.bias", "offset": 23273975808, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 23273979904, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.45.attn_norm.weight", "offset": 23278698496, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 23278731264, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.45.ffn_norm.weight", "offset": 23414980608, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 23415013376, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 23415046144, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.46.ffn_gate.weight", "offset": 23551295488, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.46.ffn_up.weight", "offset": 23687544832, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.46.ffn_norm.weight", "offset": 23823794176, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 23823826944, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 23823831040, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.46.attn_output.weight", "offset": 23828549632, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "lk.58.attn_output.", "offset": 23866298368, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 23866331136, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.46.attn_v.bias", "offset": 23904079872, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 23904083968, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.47.attn_norm.weight", "offset": 23908802560, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 23908835328, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.47.ffn_gate.weight", "offset": 24045084672, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.47.ffn_up.weight", "offset": 24181334016, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.47.ffn_norm.weight", "offset": 24317583360, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 24317616128, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 24317620224, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.47.attn_output.weight", "offset": 24322338816, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.47.attn_q.bias", "offset": 24360087552, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 24360120320, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.47.attn_v.bias", "offset": 24397869056, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 24397873152, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.48.attn_norm.weight", "offset": 24402591744, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 24402624512, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.48.ffn_gate.weight", "offset": 24538873856, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.48.ffn_up.weight", "offset": 24675123200, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.48.ffn_norm.weight", "offset": 24811372544, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 24811405312, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 24811409408, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.48.attn_output.weight", "offset": 24816128000, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.48.attn_q.bias", "offset": 24853876736, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 24853909504, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.48.attn_v.bias", "offset": 24891658240, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 24891662336, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.49.attn_norm.weight", "offset": 24896380928, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 24896413696, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.49.ffn_gate.weight", "offset": 25032663040, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.49.ffn_up.weight", "offset": 25168912384, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.49.ffn_norm.weight", "offset": 25305161728, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 25305194496, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 25305198592, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.49.attn_output.weight", "offset": 25309917184, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.49.attn_q.bias", "offset": 25347665920, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 25347698688, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.49.attn_v.bias", "offset": 25385447424, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 25385451520, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.50.attn_k.bias", "offset": 25390170112, "shape": [ 1024 ], "size": 4718592, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 25390174208, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.50.attn_output.weight", "offset": 25394892800, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.50.attn_q.bias", "offset": 25432641536, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 25432674304, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.50.attn_v.bias", "offset": 25470423040, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 25470427136, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.50.attn_norm.weight", "offset": 25475145728, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 25475178496, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.50.ffn_gate.weight", "offset": 25611427840, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.50.ffn_up.weight", "offset": 25747677184, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.50.ffn_norm.weight", "offset": 25883926528, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 25883959296, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 25883992064, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.51.ffn_gate.weight", "offset": 26020241408, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.51.ffn_up.weight", "offset": 26156490752, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.51.ffn_norm.weight", "offset": 26292740096, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 26292772864, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 26292776960, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.51.attn_output.weight", "offset": 26297495552, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.51.attn_q.bias", "offset": 26335244288, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 26335277056, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.51.attn_v.bias", "offset": 26373025792, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 26373029888, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.52.ffn_gate.weight", "offset": 26377748480, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.52.attn_k.bias", "offset": 26513997824, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 26514001920, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.52.attn_output.weight", "offset": 26518720512, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.52.attn_q.bias", "offset": 26556469248, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 26556502016, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.52.attn_v.bias", "offset": 26594250752, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 26594254848, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.52.attn_norm.weight", "offset": 26598973440, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 26599006208, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.52.ffn_up.weight", "offset": 26735255552, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.52.ffn_norm.weight", "offset": 26871504896, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 26871537664, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 26871570432, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.53.ffn_gate.weight", "offset": 27007819776, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.53.ffn_up.weight", "offset": 27144069120, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.53.ffn_norm.weight", "offset": 27280318464, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 27280351232, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 27280355328, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.53.attn_output.weight", "offset": 27285073920, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.53.attn_q.bias", "offset": 27322822656, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 27322855424, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.53.attn_v.bias", "offset": 27360604160, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 27360608256, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.54.ffn_gate.weight", "offset": 27365326848, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.54.ffn_up.weight", "offset": 27501576192, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.54.attn_k.bias", "offset": 27637825536, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 27637829632, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.54.attn_output.weight", "offset": 27642548224, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.54.attn_q.bias", "offset": 27680296960, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 27680329728, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.54.attn_v.bias", "offset": 27718078464, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 27718082560, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.54.attn_norm.weight", "offset": 27722801152, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 27722833920, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.54.ffn_norm.weight", "offset": 27859083264, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 27859116032, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 27859148800, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.55.ffn_gate.weight", "offset": 27995398144, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.55.ffn_up.weight", "offset": 28131647488, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.55.ffn_norm.weight", "offset": 28267896832, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 28267929600, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 28267933696, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.55.attn_output.weight", "offset": 28272652288, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.55.attn_q.bias", "offset": 28310401024, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 28310433792, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.55.attn_v.bias", "offset": 28348182528, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 28348186624, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.56.attn_norm.weight", "offset": 28352905216, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 28352937984, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.56.ffn_gate.weight", "offset": 28489187328, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.56.ffn_up.weight", "offset": 28625436672, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.56.ffn_norm.weight", "offset": 28761686016, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 28761718784, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 28761722880, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.56.attn_output.weight", "offset": 28766441472, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.56.attn_q.bias", "offset": 28804190208, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 28804222976, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.56.attn_v.bias", "offset": 28841971712, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 28841975808, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.57.attn_norm.weight", "offset": 28846694400, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 28846727168, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.57.ffn_gate.weight", "offset": 28982976512, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.57.ffn_up.weight", "offset": 29119225856, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.57.ffn_norm.weight", "offset": 29255475200, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 29255507968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 29255512064, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.57.attn_output.weight", "offset": 29260230656, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.57.attn_q.bias", "offset": 29297979392, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 29298012160, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.57.attn_v.bias", "offset": 29335760896, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 29335764992, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.58.attn_norm.weight", "offset": 29340483584, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 29340516352, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.58.ffn_gate.weight", "offset": 29476765696, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.58.ffn_up.weight", "offset": 29613015040, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.58.ffn_norm.weight", "offset": 29749264384, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 29749297152, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 29749301248, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.58.attn_output.weight", "offset": 29754019840, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.58.attn_q.bias", "offset": 29791768576, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 29791801344, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.58.attn_v.bias", "offset": 29829550080, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 29829554176, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.59.attn_k.bias", "offset": 29834272768, "shape": [ 1024 ], "size": 4718592, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 29834276864, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.59.attn_output.weight", "offset": 29838995456, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.59.attn_q.bias", "offset": 29876744192, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 29876776960, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.59.attn_v.bias", "offset": 29914525696, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 29914529792, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.59.attn_norm.weight", "offset": 29919248384, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 29919281152, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.59.ffn_gate.weight", "offset": 30055530496, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.59.ffn_up.weight", "offset": 30191779840, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.59.ffn_norm.weight", "offset": 30328029184, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 30328061952, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 30328094720, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.60.ffn_gate.weight", "offset": 30464344064, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.60.ffn_up.weight", "offset": 30600593408, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.60.ffn_norm.weight", "offset": 30736842752, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 30736875520, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 30736879616, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.60.attn_output.weight", "offset": 30741598208, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.60.attn_q.bias", "offset": 30779346944, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 30779379712, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.60.attn_v.bias", "offset": 30817128448, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 30817132544, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.61.ffn_gate.weight", "offset": 30821851136, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.61.attn_k.bias", "offset": 30958100480, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 30958104576, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.61.attn_output.weight", "offset": 30962823168, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.61.attn_q.bias", "offset": 31000571904, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 31000604672, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.61.attn_v.bias", "offset": 31038353408, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 31038357504, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.61.attn_norm.weight", "offset": 31043076096, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 31043108864, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.61.ffn_up.weight", "offset": 31179358208, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.61.ffn_norm.weight", "offset": 31315607552, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 31315640320, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 31315673088, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.62.ffn_gate.weight", "offset": 31451922432, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.62.ffn_up.weight", "offset": 31588171776, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.62.ffn_norm.weight", "offset": 31724421120, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 31724453888, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 31724457984, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.62.attn_output.weight", "offset": 31729176576, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.62.attn_q.bias", "offset": 31766925312, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 31766958080, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.62.attn_v.bias", "offset": 31804706816, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 31804710912, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.63.ffn_gate.weight", "offset": 31809429504, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.63.ffn_up.weight", "offset": 31945678848, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.63.attn_k.bias", "offset": 32081928192, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 32081932288, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.63.attn_output.weight", "offset": 32086650880, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.63.attn_q.bias", "offset": 32124399616, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 32124432384, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.63.attn_v.bias", "offset": 32162181120, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 32162185216, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.63.attn_norm.weight", "offset": 32166903808, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 32166936576, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.63.ffn_norm.weight", "offset": 32303185920, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 32303218688, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 32303251456, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.64.ffn_gate.weight", "offset": 32439500800, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.64.ffn_up.weight", "offset": 32575750144, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.64.ffn_norm.weight", "offset": 32711999488, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 32712032256, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 32712036352, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.64.attn_output.weight", "offset": 32716754944, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.64.attn_q.bias", "offset": 32754503680, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 32754536448, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.64.attn_v.bias", "offset": 32792285184, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 32792289280, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.65.attn_norm.weight", "offset": 32797007872, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 32797040640, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.65.ffn_gate.weight", "offset": 32933289984, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.65.ffn_up.weight", "offset": 33069539328, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.65.ffn_norm.weight", "offset": 33205788672, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 33205821440, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 33205825536, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.65.attn_output.weight", "offset": 33210544128, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.65.attn_q.bias", "offset": 33248292864, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 33248325632, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.65.attn_v.bias", "offset": 33286074368, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 33286078464, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.66.attn_norm.weight", "offset": 33290797056, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 33290829824, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.66.ffn_gate.weight", "offset": 33427079168, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.66.ffn_up.weight", "offset": 33563328512, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.66.ffn_norm.weight", "offset": 33699577856, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 33699610624, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 33699614720, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.66.attn_output.weight", "offset": 33704333312, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.66.attn_q.bias", "offset": 33742082048, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 33742114816, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.66.attn_v.bias", "offset": 33779863552, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 33779867648, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.67.attn_norm.weight", "offset": 33784586240, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 33784619008, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.67.ffn_gate.weight", "offset": 33920868352, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.67.ffn_up.weight", "offset": 34057117696, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.67.ffn_norm.weight", "offset": 34193367040, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 34193399808, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 34193403904, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.67.attn_output.weight", "offset": 34198122496, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.67.attn_q.bias", "offset": 34235871232, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 34235904000, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.67.attn_v.bias", "offset": 34273652736, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 34273656832, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.68.attn_k.bias", "offset": 34278375424, "shape": [ 1024 ], "size": 4718592, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 34278379520, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.68.attn_output.weight", "offset": 34283098112, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.68.attn_q.bias", "offset": 34320846848, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 34320879616, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.68.attn_v.bias", "offset": 34358628352, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 34358632448, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.68.attn_norm.weight", "offset": 34363351040, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 34363383808, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.68.ffn_gate.weight", "offset": 34499633152, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.68.ffn_up.weight", "offset": 34635882496, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.68.ffn_norm.weight", "offset": 34772131840, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 34772164608, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 34772197376, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.69.ffn_gate.weight", "offset": 34908446720, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.69.ffn_up.weight", "offset": 35044696064, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.69.ffn_norm.weight", "offset": 35180945408, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 35180978176, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 35180982272, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.69.attn_output.weight", "offset": 35185700864, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.69.attn_q.bias", "offset": 35223449600, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 35223482368, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.69.attn_v.bias", "offset": 35261231104, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 35261235200, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.70.ffn_gate.weight", "offset": 35265953792, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.70.attn_k.bias", "offset": 35402203136, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 35402207232, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.70.attn_output.weight", "offset": 35406925824, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.70.attn_q.bias", "offset": 35444674560, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.70.attn_q.weight", "offset": 35444707328, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "\u0014\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.70.att", "offset": 35482456064, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 35482460160, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.70.attn_norm.weight", "offset": 35487178752, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 35487211520, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.70.ffn_up.weight", "offset": 35623460864, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.70.ffn_norm.weight", "offset": 35759710208, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 35759742976, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 35759775744, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.71.ffn_gate.weight", "offset": 35896025088, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.71.ffn_up.weight", "offset": 36032274432, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.71.ffn_norm.weight", "offset": 36168523776, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 36168556544, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 36168560640, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.71.attn_output.weight", "offset": 36173279232, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.71.attn_q.bias", "offset": 36211027968, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 36211060736, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.71.attn_v.bias", "offset": 36248809472, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 36248813568, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.72.ffn_gate.weight", "offset": 36253532160, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.72.ffn_up.weight", "offset": 36389781504, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.72.attn_k.bias", "offset": 36526030848, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 36526034944, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.72.attn_output.weight", "offset": 36530753536, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.72.attn_q.bias", "offset": 36568502272, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 36568535040, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.72.attn_v.bias", "offset": 36606283776, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 36606287872, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.72.attn_norm.weight", "offset": 36611006464, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 36611039232, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.72.ffn_norm.weight", "offset": 36747288576, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 36747321344, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 36747354112, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.73.ffn_gate.weight", "offset": 36883603456, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.73.ffn_up.weight", "offset": 37019852800, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.73.ffn_norm.weight", "offset": 37156102144, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 37156134912, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 37156139008, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.73.attn_output.weight", "offset": 37160857600, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.73.attn_q.bias", "offset": 37198606336, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 37198639104, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.73.attn_v.bias", "offset": 37236387840, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 37236391936, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.74.attn_norm.weight", "offset": 37241110528, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 37241143296, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.74.ffn_gate.weight", "offset": 37377392640, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.74.ffn_up.weight", "offset": 37513641984, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.74.ffn_norm.weight", "offset": 37649891328, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 37649924096, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 37649928192, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.74.attn_output.weight", "offset": 37654646784, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.74.attn_q.bias", "offset": 37692395520, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 37692428288, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.74.attn_v.bias", "offset": 37730177024, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 37730181120, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.75.attn_norm.weight", "offset": 37734899712, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 37734932480, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.75.ffn_gate.weight", "offset": 37871181824, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.75.ffn_up.weight", "offset": 38007431168, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.75.ffn_norm.weight", "offset": 38143680512, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 38143713280, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 38143717376, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.75.attn_output.weight", "offset": 38148435968, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.75.attn_q.bias", "offset": 38186184704, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 38186217472, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.75.attn_v.bias", "offset": 38223966208, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 38223970304, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.76.attn_norm.weight", "offset": 38228688896, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 38228721664, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.76.ffn_gate.weight", "offset": 38364971008, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.76.ffn_up.weight", "offset": 38501220352, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.76.ffn_norm.weight", "offset": 38637469696, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 38637502464, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 38637506560, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.76.attn_output.weight", "offset": 38642225152, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.76.attn_q.bias", "offset": 38679973888, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 38680006656, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.76.attn_v.bias", "offset": 38717755392, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 38717759488, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.77.attn_k.bias", "offset": 38722478080, "shape": [ 1024 ], "size": 4718592, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 38722482176, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.77.attn_output.weight", "offset": 38727200768, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.77.attn_q.bias", "offset": 38764949504, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 38764982272, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.77.attn_v.bias", "offset": 38802731008, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 38802735104, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.77.attn_norm.weight", "offset": 38807453696, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 38807486464, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.77.ffn_gate.weight", "offset": 38943735808, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.77.ffn_up.weight", "offset": 39079985152, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.77.ffn_norm.weight", "offset": 39216234496, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 39216267264, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 39216300032, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.78.ffn_gate.weight", "offset": 39352549376, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.78.ffn_up.weight", "offset": 39488798720, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.78.ffn_norm.weight", "offset": 39625048064, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 39625080832, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 39625084928, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.78.attn_output.weight", "offset": 39629803520, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.78.attn_q.bias", "offset": 39667552256, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 39667585024, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.78.attn_v.bias", "offset": 39705333760, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 39705337856, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.79.ffn_gate.weight", "offset": 39710056448, "shape": [ 8192, 29568 ], "size": 4718592, "type": 2 }, { "name": "blk.79.attn_k.bias", "offset": 39846305792, "shape": [ 1024 ], "size": 136249344, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 39846309888, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "blk.79.attn_output.weight", "offset": 39851028480, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.79.attn_q.bias", "offset": 39888777216, "shape": [ 8192 ], "size": 37748736, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 39888809984, "shape": [ 8192, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.79.attn_v.bias", "offset": 39926558720, "shape": [ 1024 ], "size": 37748736, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 39926562816, "shape": [ 8192, 1024 ], "size": 4096, "type": 2 }, { "name": "output.weight", "offset": 39931281408, "shape": [ 8192, 152064 ], "size": 4718592, "type": 14 }, { "name": "blk.79.attn_norm.weight", "offset": 40953151488, "shape": [ 8192 ], "size": 1021870080, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 40953184256, "shape": [ 29568, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.79.ffn_up.weight", "offset": 41089433600, "shape": [ 8192, 29568 ], "size": 136249344, "type": 2 }, { "name": "blk.79.ffn_norm.weight", "offset": 41225682944, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "output_norm.weight", "offset": 41225715712, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }