qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 10, "general.name": "Qwen2-72B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 64, "qwen2.attention.head_count_kv": 8, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 80, "qwen2.context_length": 32768, "qwen2.embedding_length": 8192, "qwen2.feed_forward_length": 29568, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 72706203648, "tensors": [ { "name": "blk.0.ffn_gate.weight", "offset": 408748032, "shape": [ 8192, 29568 ], "size": 408748032, "type": 10 }, { "name": "blk.0.ffn_up.weight", "offset": 488226816, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.0.attn_k.bias", "offset": 567705600, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 567709696, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.0.attn_output.weight", "offset": 570462208, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.0.attn_q.bias", "offset": 599298048, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 599330816, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.0.attn_v.bias", "offset": 621350912, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 621355008, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 627122176, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 627154944, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.0.ffn_norm.weight", "offset": 763404288, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 763437056, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 763469824, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.1.ffn_gate.weight", "offset": 899719168, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.1.ffn_up.weight", "offset": 979197952, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.1.ffn_norm.weight", "offset": 1058676736, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 1058709504, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1058713600, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.1.attn_output.weight", "offset": 1061466112, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.1.attn_q.bias", "offset": 1090301952, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 1090334720, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.1.attn_v.bias", "offset": 1112354816, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 1112358912, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 1118126080, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1118158848, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.2.ffn_gate.weight", "offset": 1254408192, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.2.ffn_up.weight", "offset": 1333886976, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.2.ffn_norm.weight", "offset": 1413365760, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 1413398528, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1413402624, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.2.attn_output.weight", "offset": 1416155136, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.2.attn_q.bias", "offset": 1444990976, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 1445023744, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.2.attn_v.bias", "offset": 1467043840, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 1467047936, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 1472815104, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1472847872, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.3.ffn_gate.weight", "offset": 1609097216, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.3.ffn_up.weight", "offset": 1688576000, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.3.ffn_norm.weight", "offset": 1768054784, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 1768087552, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1768091648, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.3.attn_output.weight", "offset": 1770844160, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.3.attn_q.bias", "offset": 1799680000, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 1799712768, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.3.attn_v.bias", "offset": 1821732864, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 1821736960, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 1827504128, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1827536896, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.4.ffn_gate.weight", "offset": 1963786240, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.4.ffn_up.weight", "offset": 2043265024, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.4.ffn_norm.weight", "offset": 2122743808, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 2122776576, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 2122780672, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.4.attn_output.weight", "offset": 2125533184, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.4.attn_q.bias", "offset": 2154369024, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 2154401792, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.4.attn_v.bias", "offset": 2176421888, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 2176425984, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_k.bias", "offset": 2182193152, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 2182197248, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.5.attn_output.weight", "offset": 2184949760, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.5.attn_q.bias", "offset": 2213785600, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 2213818368, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.5.attn_v.bias", "offset": 2235838464, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 2235842560, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 2241609728, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 2241642496, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.5.ffn_gate.weight", "offset": 2377891840, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.5.ffn_up.weight", "offset": 2457370624, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.5.ffn_norm.weight", "offset": 2536849408, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 2536882176, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2536914944, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.6.ffn_gate.weight", "offset": 2673164288, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.6.ffn_up.weight", "offset": 2752643072, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.6.ffn_norm.weight", "offset": 2832121856, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 2832154624, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 2832158720, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.6.attn_output.weight", "offset": 2834911232, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.6.attn_q.bias", "offset": 2863747072, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 2863779840, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.6.attn_v.bias", "offset": 2885799936, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 2885804032, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 2891571200, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.7.attn_k.bias", "offset": 2971049984, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2971054080, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.7.attn_output.weight", "offset": 2973806592, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.7.attn_q.bias", "offset": 3002642432, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 3002675200, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.7.attn_v.bias", "offset": 3024695296, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 3024699392, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 3030466560, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 3030499328, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.7.ffn_up.weight", "offset": 3166748672, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.7.ffn_norm.weight", "offset": 3246227456, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 3246260224, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3246292992, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.8.ffn_gate.weight", "offset": 3382542336, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.8.ffn_up.weight", "offset": 3462021120, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.8.ffn_norm.weight", "offset": 3541499904, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 3541532672, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 3541536768, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.8.attn_output.weight", "offset": 3544289280, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.8.attn_q.bias", "offset": 3573125120, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 3573157888, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.8.attn_v.bias", "offset": 3595177984, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 3595182080, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 3600949248, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.9.ffn_up.weight", "offset": 3680428032, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.9.attn_k.bias", "offset": 3759906816, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 3759910912, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.9.attn_output.weight", "offset": 3762663424, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.9.attn_q.bias", "offset": 3791499264, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 3791532032, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.9.attn_v.bias", "offset": 3813552128, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 3813556224, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 3819323392, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 3819356160, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.10.ffn_gate.weight", "offset": 3955605504, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.10.ffn_up.weight", "offset": 4035084288, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.10.ffn_norm.weight", "offset": 4114563072, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 4114595840, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 4114599936, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.10.attn_output.weight", "offset": 4117352448, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.10.attn_q.bias", "offset": 4146188288, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 4146221056, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.10.attn_v.bias", "offset": 4168241152, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 4168245248, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 4174012416, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 4174045184, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.11.ffn_gate.weight", "offset": 4310294528, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.11.ffn_up.weight", "offset": 4389773312, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.11.ffn_norm.weight", "offset": 4469252096, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 4469284864, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 4469288960, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.11.attn_output.weight", "offset": 4472041472, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.11.attn_q.bias", "offset": 4500877312, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 4500910080, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.11.attn_v.bias", "offset": 4522930176, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 4522934272, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 4528701440, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 4528734208, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.9.ffn_norm.weight", "offset": 4664983552, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 4665016320, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 4665049088, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.12.ffn_gate.weight", "offset": 4801298432, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.12.ffn_up.weight", "offset": 4880777216, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.12.ffn_norm.weight", "offset": 4960256000, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 4960288768, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 4960292864, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.12.attn_output.weight", "offset": 4963045376, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.12.attn_q.bias", "offset": 4991881216, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 4991913984, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.12.attn_v.bias", "offset": 5013934080, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 5013938176, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 5019705344, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 5019738112, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.13.ffn_gate.weight", "offset": 5155987456, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.13.ffn_up.weight", "offset": 5235466240, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.13.ffn_norm.weight", "offset": 5314945024, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 5314977792, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 5314981888, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.13.attn_output.weight", "offset": 5317734400, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.13.attn_q.bias", "offset": 5346570240, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 5346603008, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.13.attn_v.bias", "offset": 5368623104, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 5368627200, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_k.bias", "offset": 5374394368, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 5374398464, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.14.attn_output.weight", "offset": 5377150976, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.14.attn_q.bias", "offset": 5405986816, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 5406019584, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.14.attn_v.bias", "offset": 5428039680, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 5428043776, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 5433810944, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 5433843712, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.14.ffn_gate.weight", "offset": 5570093056, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.14.ffn_up.weight", "offset": 5649571840, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.14.ffn_norm.weight", "offset": 5729050624, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 5729083392, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 5729116160, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.15.ffn_gate.weight", "offset": 5865365504, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.15.ffn_up.weight", "offset": 5944844288, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.15.ffn_norm.weight", "offset": 6024323072, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 6024355840, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 6024359936, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.15.attn_output.weight", "offset": 6027112448, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.15.attn_q.bias", "offset": 6055948288, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 6055981056, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.15.attn_v.bias", "offset": 6078001152, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 6078005248, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 6083772416, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.16.attn_k.bias", "offset": 6163251200, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 6163255296, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.16.attn_output.weight", "offset": 6166007808, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.16.attn_q.bias", "offset": 6194843648, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 6194876416, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.16.attn_v.bias", "offset": 6216896512, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 6216900608, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 6222667776, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 6222700544, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.16.ffn_up.weight", "offset": 6358949888, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.16.ffn_norm.weight", "offset": 6438428672, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 6438461440, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 6438494208, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.17.ffn_gate.weight", "offset": 6574743552, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.17.ffn_up.weight", "offset": 6654222336, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.17.ffn_norm.weight", "offset": 6733701120, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 6733733888, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 6733737984, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.17.attn_output.weight", "offset": 6736490496, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.17.attn_q.bias", "offset": 6765326336, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 6765359104, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.17.attn_v.bias", "offset": 6787379200, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 6787383296, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 6793150464, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.18.ffn_up.weight", "offset": 6872629248, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.18.attn_k.bias", "offset": 6952108032, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 6952112128, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.18.attn_output.weight", "offset": 6954864640, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.18.attn_q.bias", "offset": 6983700480, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 6983733248, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.18.attn_v.bias", "offset": 7005753344, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 7005757440, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 7011524608, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 7011557376, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.18.ffn_norm.weight", "offset": 7147806720, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 7147839488, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 7147872256, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.19.ffn_gate.weight", "offset": 7284121600, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.19.ffn_up.weight", "offset": 7363600384, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.19.ffn_norm.weight", "offset": 7443079168, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 7443111936, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 7443116032, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.19.attn_output.weight", "offset": 7445868544, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.19.attn_q.bias", "offset": 7474704384, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 7474737152, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.19.attn_v.bias", "offset": 7496757248, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 7496761344, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 7502528512, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 7502561280, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.20.ffn_gate.weight", "offset": 7638810624, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.20.ffn_up.weight", "offset": 7718289408, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.20.ffn_norm.weight", "offset": 7797768192, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 7797800960, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 7797805056, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.20.attn_output.weight", "offset": 7800557568, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.20.attn_q.bias", "offset": 7829393408, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 7829426176, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.20.attn_v.bias", "offset": 7851446272, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 7851450368, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 7857217536, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 7857250304, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.21.ffn_gate.weight", "offset": 7993499648, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.21.ffn_up.weight", "offset": 8072978432, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.21.ffn_norm.weight", "offset": 8152457216, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 8152489984, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 8152494080, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.21.attn_output.weight", "offset": 8155246592, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.21.attn_q.bias", "offset": 8184082432, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 8184115200, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.21.attn_v.bias", "offset": 8206135296, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 8206139392, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 8211906560, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 8211939328, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.22.ffn_gate.weight", "offset": 8348188672, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.22.ffn_up.weight", "offset": 8427667456, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.22.ffn_norm.weight", "offset": 8507146240, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 8507179008, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 8507183104, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.22.attn_output.weight", "offset": 8509935616, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.22.attn_q.bias", "offset": 8538771456, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 8538804224, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.22.attn_v.bias", "offset": 8560824320, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 8560828416, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_k.bias", "offset": 8566595584, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 8566599680, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.23.attn_output.weight", "offset": 8569352192, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.23.attn_q.bias", "offset": 8598188032, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 8598220800, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.23.attn_v.bias", "offset": 8620240896, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 8620244992, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 8626012160, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 8626044928, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.23.ffn_gate.weight", "offset": 8762294272, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.23.ffn_up.weight", "offset": 8841773056, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.23.ffn_norm.weight", "offset": 8921251840, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 8921284608, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 8921317376, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.24.ffn_gate.weight", "offset": 9057566720, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.24.ffn_up.weight", "offset": 9137045504, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.24.ffn_norm.weight", "offset": 9216524288, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 9216557056, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 9216561152, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.24.attn_output.weight", "offset": 9219313664, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.24.attn_q.bias", "offset": 9248149504, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 9248182272, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.24.attn_v.bias", "offset": 9270202368, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 9270206464, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 9275973632, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.25.attn_k.bias", "offset": 9355452416, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 9355456512, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.25.attn_output.weight", "offset": 9358209024, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.25.attn_q.bias", "offset": 9387044864, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 9387077632, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.25.attn_v.bias", "offset": 9409097728, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 9409101824, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 9414868992, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 9414901760, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.25.ffn_up.weight", "offset": 9551151104, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.25.ffn_norm.weight", "offset": 9630629888, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 9630662656, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 9630695424, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.26.ffn_gate.weight", "offset": 9766944768, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.26.ffn_up.weight", "offset": 9846423552, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.26.ffn_norm.weight", "offset": 9925902336, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 9925935104, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 9925939200, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.26.attn_output.weight", "offset": 9928691712, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.26.attn_q.bias", "offset": 9957527552, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 9957560320, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.26.attn_v.bias", "offset": 9979580416, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 9979584512, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 9985351680, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.27.ffn_up.weight", "offset": 10064830464, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.27.attn_k.bias", "offset": 10144309248, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 10144313344, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.27.attn_output.weight", "offset": 10147065856, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.27.attn_q.bias", "offset": 10175901696, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 10175934464, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.27.attn_v.bias", "offset": 10197954560, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 10197958656, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 10203725824, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 10203758592, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.27.ffn_norm.weight", "offset": 10340007936, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.28.attn_norm.weight", "offset": 10340040704, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 10340073472, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.28.ffn_gate.weight", "offset": 10476322816, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.28.ffn_up.weight", "offset": 10555801600, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.28.ffn_norm.weight", "offset": 10635280384, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 10635313152, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 10635317248, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.28.attn_output.weight", "offset": 10638069760, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.28.attn_q.bias", "offset": 10666905600, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 10666938368, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.28.attn_v.bias", "offset": 10688958464, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 10688962560, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 10694729728, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 10694762496, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.29.ffn_gate.weight", "offset": 10831011840, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.29.ffn_up.weight", "offset": 10910490624, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.29.ffn_norm.weight", "offset": 10989969408, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 10990002176, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 10990006272, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.29.attn_output.weight", "offset": 10992758784, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.29.attn_q.bias", "offset": 11021594624, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 11021627392, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.29.attn_v.bias", "offset": 11043647488, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 11043651584, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 11049418752, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 11049451520, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.30.ffn_gate.weight", "offset": 11185700864, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.30.ffn_up.weight", "offset": 11265179648, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.30.ffn_norm.weight", "offset": 11344658432, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 11344691200, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 11344695296, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.30.attn_output.weight", "offset": 11347447808, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.30.attn_q.bias", "offset": 11376283648, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 11376316416, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.30.attn_v.bias", "offset": 11398336512, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 11398340608, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 11404107776, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 11404140544, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.31.ffn_gate.weight", "offset": 11540389888, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.31.ffn_up.weight", "offset": 11619868672, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.31.ffn_norm.weight", "offset": 11699347456, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 11699380224, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 11699384320, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.31.attn_output.weight", "offset": 11702136832, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.31.attn_q.bias", "offset": 11730972672, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 11731005440, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.31.attn_v.bias", "offset": 11753025536, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 11753029632, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_k.bias", "offset": 11758796800, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 11758800896, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.32.attn_output.weight", "offset": 11761553408, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.32.attn_q.bias", "offset": 11790389248, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 11790422016, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.32.attn_v.bias", "offset": 11812442112, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.32.attn_v.weight", "offset": 11812446208, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 11818213376, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 11818246144, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.32.ffn_gate.weight", "offset": 11954495488, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.32.ffn_up.weight", "offset": 12033974272, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.32.ffn_norm.weight", "offset": 12113453056, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 12113485824, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 12113518592, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.33.ffn_gate.weight", "offset": 12249767936, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.33.ffn_up.weight", "offset": 12329246720, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.33.ffn_norm.weight", "offset": 12408725504, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.33.attn_k.bias", "offset": 12408758272, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 12408762368, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.33.attn_output.weight", "offset": 12411514880, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.33.attn_q.bias", "offset": 12440350720, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 12440383488, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.33.attn_v.bias", "offset": 12462403584, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.33.attn_v.weight", "offset": 12462407680, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.ffn_gate.weight", "offset": 12468174848, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.34.attn_k.bias", "offset": 12547653632, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 12547657728, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.34.attn_output.weight", "offset": 12550410240, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.34.attn_q.bias", "offset": 12579246080, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 12579278848, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.34.attn_v.bias", "offset": 12601298944, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.34.attn_v.weight", "offset": 12601303040, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 12607070208, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 12607102976, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.34.ffn_up.weight", "offset": 12743352320, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.34.ffn_norm.weight", "offset": 12822831104, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 12822863872, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 12822896640, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.35.ffn_gate.weight", "offset": 12959145984, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.35.ffn_up.weight", "offset": 13038624768, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.35.ffn_norm.weight", "offset": 13118103552, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.35.attn_k.bias", "offset": 13118136320, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 13118140416, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.35.attn_output.weight", "offset": 13120892928, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.35.attn_q.bias", "offset": 13149728768, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 13149761536, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.35.attn_v.bias", "offset": 13171781632, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.35.attn_v.weight", "offset": 13171785728, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 13177552896, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.36.ffn_up.weight", "offset": 13257031680, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.36.attn_k.bias", "offset": 13336510464, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 13336514560, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.36.attn_output.weight", "offset": 13339267072, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.36.attn_q.bias", "offset": 13368102912, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 13368135680, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.36.attn_v.bias", "offset": 13390155776, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.36.attn_v.weight", "offset": 13390159872, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 13395927040, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 13395959808, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.36.ffn_norm.weight", "offset": 13532209152, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 13532241920, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 13532274688, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.37.ffn_gate.weight", "offset": 13668524032, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.37.ffn_up.weight", "offset": 13748002816, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.37.ffn_norm.weight", "offset": 13827481600, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.37.attn_k.bias", "offset": 13827514368, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 13827518464, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.37.attn_output.weight", "offset": 13830270976, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.37.attn_q.bias", "offset": 13859106816, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 13859139584, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.37.attn_v.bias", "offset": 13881159680, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.37.attn_v.weight", "offset": 13881163776, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 13886930944, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 13886963712, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.38.ffn_gate.weight", "offset": 14023213056, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.38.ffn_up.weight", "offset": 14102691840, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.38.ffn_norm.weight", "offset": 14182170624, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.38.attn_k.bias", "offset": 14182203392, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 14182207488, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.38.attn_output.weight", "offset": 14184960000, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.38.attn_q.bias", "offset": 14213795840, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 14213828608, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.38.attn_v.bias", "offset": 14235848704, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.38.attn_v.weight", "offset": 14235852800, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 14241619968, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 14241652736, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.39.ffn_gate.weight", "offset": 14377902080, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.39.ffn_up.weight", "offset": 14457380864, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.39.ffn_norm.weight", "offset": 14536859648, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.39.attn_k.bias", "offset": 14536892416, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 14536896512, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.39.attn_output.weight", "offset": 14539649024, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.39.attn_q.bias", "offset": 14568484864, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 14568517632, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.39.attn_v.bias", "offset": 14590537728, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.39.attn_v.weight", "offset": 14590541824, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 14596308992, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 14596341760, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.40.ffn_gate.weight", "offset": 14732591104, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.40.ffn_up.weight", "offset": 14812069888, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.40.ffn_norm.weight", "offset": 14891548672, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.40.attn_k.bias", "offset": 14891581440, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 14891585536, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.40.attn_output.weight", "offset": 14894338048, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.40.attn_q.bias", "offset": 14923173888, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.40.attn_q.weight", "offset": 14923206656, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.40.attn_v.bias", "offset": 14945226752, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.40.attn_v.weight", "offset": 14945230848, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_k.bias", "offset": 14950998016, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 14951002112, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.41.attn_output.weight", "offset": 14953754624, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.41.attn_q.bias", "offset": 14982590464, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.41.attn_q.weight", "offset": 14982623232, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.41.attn_v.bias", "offset": 15004643328, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.41.attn_v.weight", "offset": 15004647424, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 15010414592, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 15010447360, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.41.ffn_gate.weight", "offset": 15146696704, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.41.ffn_up.weight", "offset": 15226175488, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.41.ffn_norm.weight", "offset": 15305654272, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 15305687040, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 15305719808, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.42.ffn_gate.weight", "offset": 15441969152, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.42.ffn_up.weight", "offset": 15521447936, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.42.ffn_norm.weight", "offset": 15600926720, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.42.attn_k.bias", "offset": 15600959488, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 15600963584, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.42.attn_output.weight", "offset": 15603716096, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.42.attn_q.bias", "offset": 15632551936, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.42.attn_q.weight", "offset": 15632584704, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.42.attn_v.bias", "offset": 15654604800, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.42.attn_v.weight", "offset": 15654608896, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.ffn_gate.weight", "offset": 15660376064, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.43.attn_k.bias", "offset": 15739854848, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 15739858944, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.43.attn_output.weight", "offset": 15742611456, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.43.attn_q.bias", "offset": 15771447296, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.43.attn_q.weight", "offset": 15771480064, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.43.attn_v.bias", "offset": 15793500160, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.43.attn_v.weight", "offset": 15793504256, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 15799271424, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 15799304192, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.43.ffn_up.weight", "offset": 15935553536, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.43.ffn_norm.weight", "offset": 16015032320, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 16015065088, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 16015097856, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.44.ffn_gate.weight", "offset": 16151347200, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.44.ffn_up.weight", "offset": 16230825984, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.44.ffn_norm.weight", "offset": 16310304768, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.44.attn_k.bias", "offset": 16310337536, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 16310341632, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.44.attn_output.weight", "offset": 16313094144, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.44.attn_q.bias", "offset": 16341929984, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.44.attn_q.weight", "offset": 16341962752, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.44.attn_v.bias", "offset": 16363982848, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.44.attn_v.weight", "offset": 16363986944, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.ffn_gate.weight", "offset": 16369754112, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.45.ffn_up.weight", "offset": 16449232896, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.45.attn_k.bias", "offset": 16528711680, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 16528715776, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.45.attn_output.weight", "offset": 16531468288, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.45.attn_q.bias", "offset": 16560304128, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.45.attn_q.weight", "offset": 16560336896, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.45.attn_v.bias", "offset": 16582356992, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.45.attn_v.weight", "offset": 16582361088, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 16588128256, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 16588161024, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.45.ffn_norm.weight", "offset": 16724410368, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 16724443136, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 16724475904, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.46.ffn_gate.weight", "offset": 16860725248, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.46.ffn_up.weight", "offset": 16940204032, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.46.ffn_norm.weight", "offset": 17019682816, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.46.attn_k.bias", "offset": 17019715584, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 17019719680, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.46.attn_output.weight", "offset": 17022472192, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.46.attn_q.bias", "offset": 17051308032, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.46.attn_q.weight", "offset": 17051340800, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.46.attn_v.bias", "offset": 17073360896, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.46.attn_v.weight", "offset": 17073364992, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 17079132160, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 17079164928, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.47.ffn_gate.weight", "offset": 17215414272, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.47.ffn_up.weight", "offset": 17294893056, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.47.ffn_norm.weight", "offset": 17374371840, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.47.attn_k.bias", "offset": 17374404608, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 17374408704, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.47.attn_output.weight", "offset": 17377161216, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.47.attn_q.bias", "offset": 17405997056, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.47.attn_q.weight", "offset": 17406029824, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.47.attn_v.bias", "offset": 17428049920, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.47.attn_v.weight", "offset": 17428054016, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.48.attn_norm.weight", "offset": 17433821184, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 17433853952, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.48.ffn_gate.weight", "offset": 17570103296, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.48.ffn_up.weight", "offset": 17649582080, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.48.ffn_norm.weight", "offset": 17729060864, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.48.attn_k.bias", "offset": 17729093632, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 17729097728, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.48.attn_output.weight", "offset": 17731850240, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.48.attn_q.bias", "offset": 17760686080, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.48.attn_q.weight", "offset": 17760718848, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.48.attn_v.bias", "offset": 17782738944, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.48.attn_v.weight", "offset": 17782743040, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.49.attn_norm.weight", "offset": 17788510208, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 17788542976, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.49.ffn_gate.weight", "offset": 17924792320, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.49.ffn_up.weight", "offset": 18004271104, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.49.ffn_norm.weight", "offset": 18083749888, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.49.attn_k.bias", "offset": 18083782656, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 18083786752, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.49.attn_output.weight", "offset": 18086539264, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.49.attn_q.bias", "offset": 18115375104, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.49.attn_q.weight", "offset": 18115407872, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.49.attn_v.bias", "offset": 18137427968, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.49.attn_v.weight", "offset": 18137432064, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_k.bias", "offset": 18143199232, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 18143203328, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.50.attn_output.weight", "offset": 18145955840, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.50.attn_q.bias", "offset": 18174791680, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.50.attn_q.weight", "offset": 18174824448, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.50.attn_v.bias", "offset": 18196844544, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.50.attn_v.weight", "offset": 18196848640, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.50.attn_norm.weight", "offset": 18202615808, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 18202648576, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.50.ffn_gate.weight", "offset": 18338897920, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.50.ffn_up.weight", "offset": 18418376704, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.50.ffn_norm.weight", "offset": 18497855488, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.51.attn_norm.weight", "offset": 18497888256, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 18497921024, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.51.ffn_gate.weight", "offset": 18634170368, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.51.ffn_up.weight", "offset": 18713649152, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.51.ffn_norm.weight", "offset": 18793127936, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.51.attn_k.bias", "offset": 18793160704, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 18793164800, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.51.attn_output.weight", "offset": 18795917312, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.51.attn_q.bias", "offset": 18824753152, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.51.attn_q.weight", "offset": 18824785920, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.51.attn_v.bias", "offset": 18846806016, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.51.attn_v.weight", "offset": 18846810112, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.ffn_gate.weight", "offset": 18852577280, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.52.attn_k.bias", "offset": 18932056064, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 18932060160, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.52.attn_output.weight", "offset": 18934812672, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.52.attn_q.bias", "offset": 18963648512, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.52.attn_q.weight", "offset": 18963681280, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.52.attn_v.bias", "offset": 18985701376, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.52.attn_v.weight", "offset": 18985705472, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.52.attn_norm.weight", "offset": 18991472640, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 18991505408, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.52.ffn_up.weight", "offset": 19127754752, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.52.ffn_norm.weight", "offset": 19207233536, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.53.attn_norm.weight", "offset": 19207266304, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 19207299072, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.53.ffn_gate.weight", "offset": 19343548416, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.53.ffn_up.weight", "offset": 19423027200, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.53.ffn_norm.weight", "offset": 19502505984, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.53.attn_k.bias", "offset": 19502538752, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 19502542848, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.53.attn_output.weight", "offset": 19505295360, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.53.attn_q.bias", "offset": 19534131200, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.53.attn_q.weight", "offset": 19534163968, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.53.attn_v.bias", "offset": 19556184064, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.53.attn_v.weight", "offset": 19556188160, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.ffn_gate.weight", "offset": 19561955328, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.54.ffn_up.weight", "offset": 19641434112, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.54.attn_k.bias", "offset": 19720912896, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 19720916992, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.54.attn_output.weight", "offset": 19723669504, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.54.attn_q.bias", "offset": 19752505344, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.54.attn_q.weight", "offset": 19752538112, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.54.attn_v.bias", "offset": 19774558208, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.54.attn_v.weight", "offset": 19774562304, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.54.attn_norm.weight", "offset": 19780329472, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 19780362240, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.54.ffn_norm.weight", "offset": 19916611584, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 19916644352, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 19916677120, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.55.ffn_gate.weight", "offset": 20052926464, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.55.ffn_up.weight", "offset": 20132405248, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.55.ffn_norm.weight", "offset": 20211884032, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.55.attn_k.bias", "offset": 20211916800, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 20211920896, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.55.attn_output.weight", "offset": 20214673408, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.55.attn_q.bias", "offset": 20243509248, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.55.attn_q.weight", "offset": 20243542016, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.55.attn_v.bias", "offset": 20265562112, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.55.attn_v.weight", "offset": 20265566208, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.56.attn_norm.weight", "offset": 20271333376, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 20271366144, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.56.ffn_gate.weight", "offset": 20407615488, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.56.ffn_up.weight", "offset": 20487094272, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.56.ffn_norm.weight", "offset": 20566573056, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.56.attn_k.bias", "offset": 20566605824, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 20566609920, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.56.attn_output.weight", "offset": 20569362432, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.56.attn_q.bias", "offset": 20598198272, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.56.attn_q.weight", "offset": 20598231040, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.56.attn_v.bias", "offset": 20620251136, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.56.attn_v.weight", "offset": 20620255232, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.57.attn_norm.weight", "offset": 20626022400, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 20626055168, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.57.ffn_gate.weight", "offset": 20762304512, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.57.ffn_up.weight", "offset": 20841783296, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.57.ffn_norm.weight", "offset": 20921262080, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.57.attn_k.bias", "offset": 20921294848, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 20921298944, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.57.attn_output.weight", "offset": 20924051456, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.57.attn_q.bias", "offset": 20952887296, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.57.attn_q.weight", "offset": 20952920064, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.57.attn_v.bias", "offset": 20974940160, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.57.attn_v.weight", "offset": 20974944256, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.58.attn_norm.weight", "offset": 20980711424, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 20980744192, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.58.ffn_gate.weight", "offset": 21116993536, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.58.ffn_up.weight", "offset": 21196472320, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.58.ffn_norm.weight", "offset": 21275951104, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.58.attn_k.bias", "offset": 21275983872, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 21275987968, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.58.attn_output.weight", "offset": 21278740480, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.58.attn_q.bias", "offset": 21307576320, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.58.attn_q.weight", "offset": 21307609088, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.58.attn_v.bias", "offset": 21329629184, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.58.attn_v.weight", "offset": 21329633280, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_k.bias", "offset": 21335400448, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 21335404544, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.59.attn_output.weight", "offset": 21338157056, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.59.attn_q.bias", "offset": 21366992896, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.59.attn_q.weight", "offset": 21367025664, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.59.attn_v.bias", "offset": 21389045760, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.59.attn_v.weight", "offset": 21389049856, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.59.attn_norm.weight", "offset": 21394817024, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 21394849792, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.59.ffn_gate.weight", "offset": 21531099136, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.59.ffn_up.weight", "offset": 21610577920, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.59.ffn_norm.weight", "offset": 21690056704, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.60.attn_norm.weight", "offset": 21690089472, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 21690122240, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.60.ffn_gate.weight", "offset": 21826371584, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.60.ffn_up.weight", "offset": 21905850368, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.60.ffn_norm.weight", "offset": 21985329152, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.60.attn_k.bias", "offset": 21985361920, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 21985366016, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.60.attn_output.weight", "offset": 21988118528, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.60.attn_q.bias", "offset": 22016954368, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.60.attn_q.weight", "offset": 22016987136, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.60.attn_v.bias", "offset": 22039007232, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.60.attn_v.weight", "offset": 22039011328, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.ffn_gate.weight", "offset": 22044778496, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.61.attn_k.bias", "offset": 22124257280, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 22124261376, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.61.attn_output.weight", "offset": 22127013888, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.61.attn_q.bias", "offset": 22155849728, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.61.attn_q.weight", "offset": 22155882496, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.61.attn_v.bias", "offset": 22177902592, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.61.attn_v.weight", "offset": 22177906688, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.61.attn_norm.weight", "offset": 22183673856, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 22183706624, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.61.ffn_up.weight", "offset": 22319955968, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.61.ffn_norm.weight", "offset": 22399434752, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.62.attn_norm.weight", "offset": 22399467520, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 22399500288, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.62.ffn_gate.weight", "offset": 22535749632, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.62.ffn_up.weight", "offset": 22615228416, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.62.ffn_norm.weight", "offset": 22694707200, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.62.attn_k.bias", "offset": 22694739968, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 22694744064, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.62.attn_output.weight", "offset": 22697496576, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.62.attn_q.bias", "offset": 22726332416, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.62.attn_q.weight", "offset": 22726365184, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.62.attn_v.bias", "offset": 22748385280, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.62.attn_v.weight", "offset": 22748389376, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.ffn_gate.weight", "offset": 22754156544, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.63.ffn_up.weight", "offset": 22833635328, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.63.attn_k.bias", "offset": 22913114112, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 22913118208, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.63.attn_output.weight", "offset": 22915870720, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.63.attn_q.bias", "offset": 22944706560, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.63.attn_q.weight", "offset": 22944739328, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.63.attn_v.bias", "offset": 22966759424, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.63.attn_v.weight", "offset": 22966763520, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.63.attn_norm.weight", "offset": 22972530688, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 22972563456, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.63.ffn_norm.weight", "offset": 23108812800, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.64.attn_norm.weight", "offset": 23108845568, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 23108878336, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.64.ffn_gate.weight", "offset": 23245127680, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.64.ffn_up.weight", "offset": 23324606464, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.64.ffn_norm.weight", "offset": 23404085248, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.64.attn_k.bias", "offset": 23404118016, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 23404122112, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.64.attn_output.weight", "offset": 23406874624, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.64.attn_q.bias", "offset": 23435710464, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.64.attn_q.weight", "offset": 23435743232, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.64.attn_v.bias", "offset": 23457763328, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.64.attn_v.weight", "offset": 23457767424, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.65.attn_norm.weight", "offset": 23463534592, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 23463567360, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.65.ffn_gate.weight", "offset": 23599816704, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.65.ffn_up.weight", "offset": 23679295488, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.65.ffn_norm.weight", "offset": 23758774272, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.65.attn_k.bias", "offset": 23758807040, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 23758811136, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.65.attn_output.weight", "offset": 23761563648, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.65.attn_q.bias", "offset": 23790399488, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.65.attn_q.weight", "offset": 23790432256, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.65.attn_v.bias", "offset": 23812452352, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.65.attn_v.weight", "offset": 23812456448, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.66.attn_norm.weight", "offset": 23818223616, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 23818256384, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.66.ffn_gate.weight", "offset": 23954505728, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.66.ffn_up.weight", "offset": 24033984512, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.66.ffn_norm.weight", "offset": 24113463296, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.66.attn_k.bias", "offset": 24113496064, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 24113500160, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.66.attn_output.weight", "offset": 24116252672, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.66.attn_q.bias", "offset": 24145088512, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.66.attn_q.weight", "offset": 24145121280, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.66.attn_v.bias", "offset": 24167141376, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.66.attn_v.weight", "offset": 24167145472, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.67.attn_norm.weight", "offset": 24172912640, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 24172945408, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.67.ffn_gate.weight", "offset": 24309194752, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.67.ffn_up.weight", "offset": 24388673536, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.67.ffn_norm.weight", "offset": 24468152320, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.67.attn_k.bias", "offset": 24468185088, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 24468189184, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.67.attn_output.weight", "offset": 24470941696, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.67.attn_q.bias", "offset": 24499777536, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.67.attn_q.weight", "offset": 24499810304, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.67.attn_v.bias", "offset": 24521830400, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.67.attn_v.weight", "offset": 24521834496, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_k.bias", "offset": 24527601664, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.68.attn_k.weight", "offset": 24527605760, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.68.attn_output.weight", "offset": 24530358272, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.68.attn_q.bias", "offset": 24559194112, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.68.attn_q.weight", "offset": 24559226880, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.68.attn_v.bias", "offset": 24581246976, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.68.attn_v.weight", "offset": 24581251072, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.68.attn_norm.weight", "offset": 24587018240, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 24587051008, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.68.ffn_gate.weight", "offset": 24723300352, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.68.ffn_up.weight", "offset": 24802779136, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.68.ffn_norm.weight", "offset": 24882257920, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 24882290688, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 24882323456, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.69.ffn_gate.weight", "offset": 25018572800, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.69.ffn_up.weight", "offset": 25098051584, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.69.ffn_norm.weight", "offset": 25177530368, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.69.attn_k.bias", "offset": 25177563136, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 25177567232, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.69.attn_output.weight", "offset": 25180319744, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.69.attn_q.bias", "offset": 25209155584, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.69.attn_q.weight", "offset": 25209188352, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.69.attn_v.bias", "offset": 25231208448, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.69.attn_v.weight", "offset": 25231212544, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.ffn_gate.weight", "offset": 25236979712, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.70.attn_k.bias", "offset": 25316458496, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 25316462592, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.70.attn_output.weight", "offset": 25319215104, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.70.attn_q.bias", "offset": 25348050944, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "\ufffdj@\ufffdZK\ufffdj\ufffd'\ufffd\u0026\u0026g\ufffd%.ab6", "offset": 25348083712, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.70.attn_v.bias", "offset": 25370103808, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.70.attn_v.weight", "offset": 25370107904, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.70.attn_norm.weight", "offset": 25375875072, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 25375907840, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.70.ffn_up.weight", "offset": 25512157184, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.70.ffn_norm.weight", "offset": 25591635968, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.71.attn_norm.weight", "offset": 25591668736, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 25591701504, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.71.ffn_gate.weight", "offset": 25727950848, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.71.ffn_up.weight", "offset": 25807429632, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.71.ffn_norm.weight", "offset": 25886908416, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.71.attn_k.bias", "offset": 25886941184, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 25886945280, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.71.attn_output.weight", "offset": 25889697792, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.71.attn_q.bias", "offset": 25918533632, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.71.attn_q.weight", "offset": 25918566400, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.71.attn_v.bias", "offset": 25940586496, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.71.attn_v.weight", "offset": 25940590592, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.ffn_gate.weight", "offset": 25946357760, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.72.ffn_up.weight", "offset": 26025836544, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.72.attn_k.bias", "offset": 26105315328, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 26105319424, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.72.attn_output.weight", "offset": 26108071936, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.72.attn_q.bias", "offset": 26136907776, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.72.attn_q.weight", "offset": 26136940544, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.72.attn_v.bias", "offset": 26158960640, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.72.attn_v.weight", "offset": 26158964736, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.72.attn_norm.weight", "offset": 26164731904, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 26164764672, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.72.ffn_norm.weight", "offset": 26301014016, "shape": [ 8192 ], "size": 136249344, "type": 0 }, { "name": "blk.73.attn_norm.weight", "offset": 26301046784, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 26301079552, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.73.ffn_gate.weight", "offset": 26437328896, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.73.ffn_up.weight", "offset": 26516807680, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.73.ffn_norm.weight", "offset": 26596286464, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.73.attn_k.bias", "offset": 26596319232, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 26596323328, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.73.attn_output.weight", "offset": 26599075840, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.73.attn_q.bias", "offset": 26627911680, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.73.attn_q.weight", "offset": 26627944448, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.73.attn_v.bias", "offset": 26649964544, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.73.attn_v.weight", "offset": 26649968640, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.74.attn_norm.weight", "offset": 26655735808, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 26655768576, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.74.ffn_gate.weight", "offset": 26792017920, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.74.ffn_up.weight", "offset": 26871496704, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.74.ffn_norm.weight", "offset": 26950975488, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.74.attn_k.bias", "offset": 26951008256, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.74.attn_k.weight", "offset": 26951012352, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.74.attn_output.weight", "offset": 26953764864, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.74.attn_q.bias", "offset": 26982600704, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.74.attn_q.weight", "offset": 26982633472, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.74.attn_v.bias", "offset": 27004653568, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.74.attn_v.weight", "offset": 27004657664, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.75.attn_norm.weight", "offset": 27010424832, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 27010457600, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.75.ffn_gate.weight", "offset": 27146706944, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.75.ffn_up.weight", "offset": 27226185728, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.75.ffn_norm.weight", "offset": 27305664512, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.75.attn_k.bias", "offset": 27305697280, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 27305701376, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.75.attn_output.weight", "offset": 27308453888, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.75.attn_q.bias", "offset": 27337289728, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.75.attn_q.weight", "offset": 27337322496, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.75.attn_v.bias", "offset": 27359342592, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.75.attn_v.weight", "offset": 27359346688, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.76.attn_norm.weight", "offset": 27365113856, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 27365146624, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.76.ffn_gate.weight", "offset": 27501395968, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.76.ffn_up.weight", "offset": 27580874752, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.76.ffn_norm.weight", "offset": 27660353536, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.76.attn_k.bias", "offset": 27660386304, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 27660390400, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.76.attn_output.weight", "offset": 27663142912, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.76.attn_q.bias", "offset": 27691978752, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.76.attn_q.weight", "offset": 27692011520, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.76.attn_v.bias", "offset": 27714031616, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.76.attn_v.weight", "offset": 27714035712, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_k.bias", "offset": 27719802880, "shape": [ 1024 ], "size": 5767168, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 27719806976, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.77.attn_output.weight", "offset": 27722559488, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.77.attn_q.bias", "offset": 27751395328, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.77.attn_q.weight", "offset": 27751428096, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.77.attn_v.bias", "offset": 27773448192, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.77.attn_v.weight", "offset": 27773452288, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.77.attn_norm.weight", "offset": 27779219456, "shape": [ 8192 ], "size": 5767168, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 27779252224, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.77.ffn_gate.weight", "offset": 27915501568, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.77.ffn_up.weight", "offset": 27994980352, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.77.ffn_norm.weight", "offset": 28074459136, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.78.attn_norm.weight", "offset": 28074491904, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 28074524672, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.78.ffn_gate.weight", "offset": 28210774016, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.78.ffn_up.weight", "offset": 28290252800, "shape": [ 8192, 29568 ], "size": 79478784, "type": 10 }, { "name": "blk.78.ffn_norm.weight", "offset": 28369731584, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "blk.78.attn_k.bias", "offset": 28369764352, "shape": [ 1024 ], "size": 32768, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 28369768448, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.78.attn_output.weight", "offset": 28372520960, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.78.attn_q.bias", "offset": 28401356800, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.78.attn_q.weight", "offset": 28401389568, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.78.attn_v.bias", "offset": 28423409664, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.78.attn_v.weight", "offset": 28423413760, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "blk.79.ffn_gate.weight", "offset": 28429180928, "shape": [ 8192, 29568 ], "size": 5767168, "type": 10 }, { "name": "blk.79.attn_k.bias", "offset": 28508659712, "shape": [ 1024 ], "size": 79478784, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 28508663808, "shape": [ 8192, 1024 ], "size": 4096, "type": 10 }, { "name": "blk.79.attn_output.weight", "offset": 28511416320, "shape": [ 8192, 8192 ], "size": 2752512, "type": 11 }, { "name": "blk.79.attn_q.bias", "offset": 28540252160, "shape": [ 8192 ], "size": 28835840, "type": 0 }, { "name": "blk.79.attn_q.weight", "offset": 28540284928, "shape": [ 8192, 8192 ], "size": 32768, "type": 10 }, { "name": "blk.79.attn_v.bias", "offset": 28562305024, "shape": [ 1024 ], "size": 22020096, "type": 0 }, { "name": "blk.79.attn_v.weight", "offset": 28562309120, "shape": [ 8192, 1024 ], "size": 4096, "type": 13 }, { "name": "output.weight", "offset": 28568076288, "shape": [ 8192, 152064 ], "size": 5767168, "type": 14 }, { "name": "blk.79.attn_norm.weight", "offset": 29589946368, "shape": [ 8192 ], "size": 1021870080, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 29589979136, "shape": [ 29568, 8192 ], "size": 32768, "type": 20 }, { "name": "blk.79.ffn_up.weight", "offset": 29726228480, "shape": [ 8192, 29568 ], "size": 136249344, "type": 10 }, { "name": "blk.79.ffn_norm.weight", "offset": 29805707264, "shape": [ 8192 ], "size": 79478784, "type": 0 }, { "name": "output_norm.weight", "offset": 29805740032, "shape": [ 8192 ], "size": 32768, "type": 0 } ], "version": 3 }