codestral

{ "metadata": { "general.architecture": "llama", "general.file_type": 12, "general.name": "Codestral-22B-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 32768, "llama.embedding_length": 6144, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_space_prefix": true, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 22247282688, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 86507520, "shape": [ 6144 ], "size": 86507520, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 86532096, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 155738112, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 198991872, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 242245632, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 242270208, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 244973568, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 266207232, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 282427392, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 286752768, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 286777344, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 355983360, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 399237120, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 442490880, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 442515456, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 445218816, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 466452480, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 482672640, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 486998016, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 487022592, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 556228608, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 599482368, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 642736128, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "lk.17.attn_v.weight", "offset": 642760704, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 645464064, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 666697728, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.2.attn_v.weight", "offset": 682917888, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 686456832, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 686481408, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 743104512, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 786358272, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 829612032, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 829636608, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 832339968, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 853573632, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.3.attn_v.weight", "offset": 869793792, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 873332736, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 873357312, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 929980416, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 973234176, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 1016487936, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1016512512, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 1019215872, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 1040449536, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 1056669696, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 1060208640, "shape": [ 6144, 16384 ], "size": 3538944, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 1103462400, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.5.attn_k.weight", "offset": 1146716160, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 1149419520, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 1170653184, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 1186873344, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1190412288, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1190436864, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.10.ffn_gate.weight", "offset": 1247059968, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 1290313728, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 1333567488, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1333592064, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 1336295424, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.10.attn_q.weight", "offset": 1357529088, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 1373749248, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1377288192, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1377312768, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 1433935872, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1477189632, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 1520443392, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1520467968, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 1523171328, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.11.attn_q.weight", "offset": 1544404992, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.11.attn_v.weight", "offset": 1560625152, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.12.attn_k.weight", "offset": 1564164096, "shape": [ 6144, 1024 ], "size": 3538944, "type": 11 }, { "name": "blk.12.attn_q.weight", "offset": 1566867456, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.12.attn_v.weight", "offset": 1583087616, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 1586626560, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1586651136, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 1643274240, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 1643298816, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1643323392, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 1699946496, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 1743200256, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 1786454016, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1786478592, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 1789181952, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 1810415616, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 1826635776, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 1830174720, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1830199296, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 1886822400, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 1930076160, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 1973329920, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1973354496, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 1976057856, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 1997291520, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 2013511680, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 2017050624, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2017075200, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 2073698304, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 2116952064, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 2160205824, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2160230400, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 2162933760, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 2184167424, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 2200387584, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 2203926528, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2203951104, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 2260574208, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 2303827968, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 2347081728, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2347106304, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 2349809664, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 2371043328, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.9.attn_v.weight", "offset": 2387263488, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 2390802432, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2390827008, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 2447450112, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 2490703872, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 2533957632, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2533982208, "shape": [ 6144, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 2555215872, "shape": [ 6144 ], "size": 21233664, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2555240448, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 2611863552, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 2655117312, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 2698371072, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2698395648, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 2701099008, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.13.attn_q.weight", "offset": 2722332672, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.13.attn_v.weight", "offset": 2738552832, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 2742091776, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2742116352, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 2798739456, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 2841993216, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 2885246976, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2885271552, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 2887974912, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 2909208576, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 2925428736, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 2928967680, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2928992256, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 2985615360, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 3028869120, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 3072122880, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 3072147456, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 3074850816, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 3096084480, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 3112304640, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 3115843584, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3115868160, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 3172491264, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 3215745024, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 3258998784, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 3259023360, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 3261726720, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.16.attn_q.weight", "offset": 3282960384, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 3299180544, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 3302719488, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3302744064, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 3359367168, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 3402620928, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 3445874688, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 3445899264, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 3448602624, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 3469836288, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "33.ffn_down.weight\u0002\u0000", "offset": 3486056448, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 3489595392, "shape": [ 6144, 16384 ], "size": 3538944, "type": 11 }, { "name": "blk.18.attn_k.weight", "offset": 3532849152, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 3535552512, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.18.attn_q.weight", "offset": 3556786176, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 3573006336, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 3576545280, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3576569856, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 3633192960, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 3676446720, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 3676471296, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3676495872, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 3733118976, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 3776372736, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 3819626496, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 3819651072, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 3822354432, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 3843588096, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 3859808256, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 3863347200, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3863371776, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 3919994880, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 3963248640, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 4006502400, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 4006526976, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 4009230336, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 4030464000, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 4046684160, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 4050223104, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4050247680, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 4106870784, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 4150124544, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 4193378304, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 4193402880, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 4196106240, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 4217339904, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 4233560064, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 4237099008, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4237123584, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 4293746688, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 4337000448, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 4380254208, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 4380278784, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 4382982144, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 4404215808, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 4420435968, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 4423974912, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4423999488, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 4480622592, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 4523876352, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 4567130112, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 4567154688, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 4569858048, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 4591091712, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 4607311872, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 4610850816, "shape": [ 6144, 16384 ], "size": 3538944, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 4654104576, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.24.attn_k.weight", "offset": 4697358336, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 4700061696, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 4721295360, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 4737515520, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 4741054464, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4741079040, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 4797702144, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.25.attn_norm.weight", "offset": 4797726720, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4797751296, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 4854374400, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 4897628160, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 4940881920, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 4940906496, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 4943609856, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 4964843520, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 4981063680, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 4984602624, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4984627200, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 5041250304, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 5084504064, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 5127757824, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 5127782400, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 5130485760, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 5151719424, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 5167939584, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 5171478528, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5171503104, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 5228126208, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 5271379968, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 5314633728, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 5314658304, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 5317361664, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.27.attn_q.weight", "offset": 5338595328, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 5354815488, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 5358354432, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5358379008, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.28.ffn_gate.weight", "offset": 5415002112, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 5458255872, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 5501509632, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 5501534208, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 5504237568, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.28.attn_q.weight", "offset": 5525471232, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 5541691392, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 5545230336, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 5545254912, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.29.ffn_gate.weight", "offset": 5601878016, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 5645131776, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 5688385536, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 5688410112, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 5691113472, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.29.attn_q.weight", "offset": 5712347136, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 5728567296, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 5732106240, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 5732130816, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.30.ffn_gate.weight", "offset": 5788753920, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 5832007680, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 5875261440, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 5875286016, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 5877989376, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.30.attn_q.weight", "offset": 5899223040, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 5915443200, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.31.attn_k.weight", "offset": 5918982144, "shape": [ 6144, 1024 ], "size": 3538944, "type": 11 }, { "name": "blk.31.attn_q.weight", "offset": 5921685504, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 5937905664, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 5941444608, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 5941469184, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.31.ffn_gate.weight", "offset": 5998092288, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 6041346048, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 6084599808, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 6084624384, "shape": [ 6144, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 6105858048, "shape": [ 6144 ], "size": 21233664, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 6105882624, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.32.ffn_gate.weight", "offset": 6162505728, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 6205759488, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 6249013248, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 6249037824, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 6251741184, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.32.attn_q.weight", "offset": 6272974848, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.32.attn_v.weight", "offset": 6289195008, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 6292733952, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6292758528, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.33.ffn_gate.weight", "offset": 6349381632, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 6392635392, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 6435889152, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 6435913728, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 6438617088, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.33.attn_q.weight", "offset": 6459850752, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.33.attn_v.weight", "offset": 6476070912, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 6479609856, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 6479634432, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.34.ffn_gate.weight", "offset": 6536257536, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 6579511296, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 6622765056, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 6622789632, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 6625492992, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.34.attn_q.weight", "offset": 6646726656, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.34.attn_v.weight", "offset": 6662946816, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.35.attn_norm.weight", "offset": 6666485760, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 6666510336, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.35.ffn_gate.weight", "offset": 6723133440, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 6766387200, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 6809640960, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 6809665536, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 6812368896, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.35.attn_q.weight", "offset": 6833602560, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.35.attn_v.weight", "offset": 6849822720, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.36.attn_norm.weight", "offset": 6853361664, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 6853386240, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.36.ffn_gate.weight", "offset": 6910009344, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 6953263104, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 6996516864, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 6996541440, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 6999244800, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.36.attn_q.weight", "offset": 7020478464, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.36.attn_v.weight", "offset": 7036698624, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.37.ffn_gate.weight", "offset": 7040237568, "shape": [ 6144, 16384 ], "size": 3538944, "type": 11 }, { "name": "blk.37.attn_k.weight", "offset": 7083491328, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 7086194688, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.37.attn_q.weight", "offset": 7107428352, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.37.attn_v.weight", "offset": 7123648512, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 7127187456, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 7127212032, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 7183835136, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 7227088896, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.38.attn_norm.weight", "offset": 7227113472, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 7227138048, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.38.ffn_gate.weight", "offset": 7283761152, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 7327014912, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 7370268672, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 7370293248, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 7372996608, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.38.attn_q.weight", "offset": 7394230272, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.38.attn_v.weight", "offset": 7410450432, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 7413989376, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 7414013952, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.39.ffn_gate.weight", "offset": 7470637056, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 7513890816, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 7557144576, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 7557169152, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 7559872512, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.39.attn_q.weight", "offset": 7581106176, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.39.attn_v.weight", "offset": 7597326336, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.40.attn_norm.weight", "offset": 7600865280, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 7600889856, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.40.ffn_gate.weight", "offset": 7657512960, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.40.ffn_up.weight", "offset": 7700766720, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.40.ffn_norm.weight", "offset": 7744020480, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 7744045056, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 7746748416, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.40.attn_q.weight", "offset": 7767982080, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.40.attn_v.weight", "offset": 7784202240, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.41.attn_norm.weight", "offset": 7787741184, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 7787765760, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.41.ffn_gate.weight", "offset": 7844388864, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.41.ffn_up.weight", "offset": 7887642624, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.41.ffn_norm.weight", "offset": 7930896384, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 7930920960, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 7933624320, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.41.attn_q.weight", "offset": 7954857984, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.41.attn_v.weight", "offset": 7971078144, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.42.attn_norm.weight", "offset": 7974617088, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 7974641664, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.42.ffn_gate.weight", "offset": 8031264768, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.42.ffn_up.weight", "offset": 8074518528, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.42.ffn_norm.weight", "offset": 8117772288, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 8117796864, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.42.attn_output.weight", "offset": 8120500224, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.42.attn_q.weight", "offset": 8141733888, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.42.attn_v.weight", "offset": 8157954048, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.43.ffn_gate.weight", "offset": 8161492992, "shape": [ 6144, 16384 ], "size": 3538944, "type": 11 }, { "name": "blk.43.ffn_up.weight", "offset": 8204746752, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.43.attn_k.weight", "offset": 8248000512, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.43.attn_output.weight", "offset": 8250703872, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.43.attn_q.weight", "offset": 8271937536, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.43.attn_v.weight", "offset": 8288157696, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.43.attn_norm.weight", "offset": 8291696640, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 8291721216, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.43.ffn_norm.weight", "offset": 8348344320, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 8348368896, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 8348393472, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.44.ffn_gate.weight", "offset": 8405016576, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.44.ffn_up.weight", "offset": 8448270336, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.44.ffn_norm.weight", "offset": 8491524096, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 8491548672, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.44.attn_output.weight", "offset": 8494252032, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.44.attn_q.weight", "offset": 8515485696, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.44.attn_v.weight", "offset": 8531705856, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.45.attn_norm.weight", "offset": 8535244800, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 8535269376, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.45.ffn_gate.weight", "offset": 8591892480, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.45.ffn_up.weight", "offset": 8635146240, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.45.ffn_norm.weight", "offset": 8678400000, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 8678424576, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.45.attn_output.weight", "offset": 8681127936, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.45.attn_q.weight", "offset": 8702361600, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.45.attn_v.weight", "offset": 8718581760, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.46.attn_norm.weight", "offset": 8722120704, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 8722145280, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.46.ffn_gate.weight", "offset": 8778768384, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.46.ffn_up.weight", "offset": 8822022144, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.46.ffn_norm.weight", "offset": 8865275904, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 8865300480, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.46.attn_output.weight", "offset": 8868003840, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.46.attn_q.weight", "offset": 8889237504, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.46.attn_v.weight", "offset": 8905457664, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.47.attn_norm.weight", "offset": 8908996608, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 8909021184, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.47.ffn_gate.weight", "offset": 8965644288, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.47.ffn_up.weight", "offset": 9008898048, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.47.ffn_norm.weight", "offset": 9052151808, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 9052176384, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.47.attn_output.weight", "offset": 9054879744, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.47.attn_q.weight", "offset": 9076113408, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.47.attn_v.weight", "offset": 9092333568, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.48.attn_norm.weight", "offset": 9095872512, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 9095897088, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.48.ffn_gate.weight", "offset": 9152520192, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "\ufffd\f\u0000#\ufffd/ѽ,\u0026\ufffd\ufffdS\ufffd\ufffd\ufffd\u00118\ufffd\ufffd", "offset": 9195773952, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.48.ffn_norm.weight", "offset": 9239027712, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 9239052288, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.48.attn_output.weight", "offset": 9241755648, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.48.attn_q.weight", "offset": 9262989312, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.48.attn_v.weight", "offset": 9279209472, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.49.attn_norm.weight", "offset": 9282748416, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 9282772992, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.49.ffn_gate.weight", "offset": 9339396096, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.49.ffn_up.weight", "offset": 9382649856, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.49.ffn_norm.weight", "offset": 9425903616, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 9425928192, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.49.attn_output.weight", "offset": 9428631552, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.49.attn_q.weight", "offset": 9449865216, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.49.attn_v.weight", "offset": 9466085376, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.50.attn_k.weight", "offset": 9469624320, "shape": [ 6144, 1024 ], "size": 3538944, "type": 11 }, { "name": "blk.50.attn_q.weight", "offset": 9472327680, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.50.attn_v.weight", "offset": 9488547840, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "output.weight", "offset": 9492086784, "shape": [ 6144, 32768 ], "size": 3538944, "type": 14 }, { "name": "blk.50.attn_norm.weight", "offset": 9657237504, "shape": [ 6144 ], "size": 165150720, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 9657262080, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.50.ffn_gate.weight", "offset": 9713885184, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.50.ffn_up.weight", "offset": 9757138944, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.50.ffn_norm.weight", "offset": 9800392704, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 9800417280, "shape": [ 6144, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.51.attn_norm.weight", "offset": 9821650944, "shape": [ 6144 ], "size": 21233664, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 9821675520, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.51.ffn_gate.weight", "offset": 9878298624, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.51.ffn_up.weight", "offset": 9921552384, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.51.ffn_norm.weight", "offset": 9964806144, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 9964830720, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.51.attn_output.weight", "offset": 9967534080, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.51.attn_q.weight", "offset": 9988767744, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.51.attn_v.weight", "offset": 10004987904, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.52.attn_norm.weight", "offset": 10008526848, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 10008551424, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.52.ffn_gate.weight", "offset": 10065174528, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.52.ffn_up.weight", "offset": 10108428288, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.52.ffn_norm.weight", "offset": 10151682048, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 10151706624, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.52.attn_output.weight", "offset": 10154409984, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.52.attn_q.weight", "offset": 10175643648, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.52.attn_v.weight", "offset": 10191863808, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.53.attn_norm.weight", "offset": 10195402752, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 10195427328, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.53.ffn_gate.weight", "offset": 10252050432, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.53.ffn_up.weight", "offset": 10295304192, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.53.ffn_norm.weight", "offset": 10338557952, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 10338582528, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.53.attn_output.weight", "offset": 10341285888, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.53.attn_q.weight", "offset": 10362519552, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.53.attn_v.weight", "offset": 10378739712, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.54.attn_norm.weight", "offset": 10382278656, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 10382303232, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.54.ffn_gate.weight", "offset": 10438926336, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.54.ffn_up.weight", "offset": 10482180096, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.54.ffn_norm.weight", "offset": 10525433856, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 10525458432, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.54.attn_output.weight", "offset": 10528161792, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.54.attn_q.weight", "offset": 10549395456, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.54.attn_v.weight", "offset": 10565615616, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "blk.55.attn_norm.weight", "offset": 10569154560, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 10569179136, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.55.ffn_gate.weight", "offset": 10625802240, "shape": [ 6144, 16384 ], "size": 56623104, "type": 11 }, { "name": "blk.55.ffn_up.weight", "offset": 10669056000, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.55.ffn_norm.weight", "offset": 10712309760, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 10712334336, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.55.attn_output.weight", "offset": 10715037696, "shape": [ 6144, 6144 ], "size": 2703360, "type": 12 }, { "name": "blk.55.attn_q.weight", "offset": 10736271360, "shape": [ 6144, 6144 ], "size": 21233664, "type": 11 }, { "name": "blk.55.attn_v.weight", "offset": 10752491520, "shape": [ 6144, 1024 ], "size": 16220160, "type": 12 }, { "name": "output_norm.weight", "offset": 10756030464, "shape": [ 6144 ], "size": 3538944, "type": 0 } ], "version": 3 }