codestral

{ "metadata": { "general.architecture": "llama", "general.file_type": 13, "general.name": "Codestral-22B-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 32768, "llama.embedding_length": 6144, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_space_prefix": true, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 22247282688, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 86507520, "shape": [ 6144 ], "size": 86507520, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 86532096, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 155738112, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 198991872, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 242245632, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 242270208, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 244973568, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 270925824, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 287145984, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 291471360, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 291495936, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 360701952, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 403955712, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 447209472, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 447234048, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 449937408, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 475889664, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 492109824, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 496435200, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 496459776, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 565665792, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 608919552, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 652173312, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 652197888, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 654901248, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 680853504, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.2.attn_v.weight", "offset": 697073664, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 701399040, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 701423616, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 770629632, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 813883392, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 857137152, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 857161728, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 859865088, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 885817344, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.3.attn_v.weight", "offset": 902037504, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 906362880, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 906387456, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 975593472, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 1018847232, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 1062100992, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1062125568, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 1064828928, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 1090781184, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 1107001344, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 1111326720, "shape": [ 6144, 16384 ], "size": 4325376, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 1154580480, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.5.attn_k.weight", "offset": 1197834240, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 1200537600, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 1226489856, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 1242710016, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 1247035392, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1247059968, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 1316265984, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 1359519744, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 1402773504, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1402798080, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 1405501440, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 1431453696, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 1447673856, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 1451999232, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1452023808, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 1521229824, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1564483584, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 1607737344, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1607761920, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 1610465280, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 1636417536, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.11.attn_v.weight", "offset": 1652637696, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.12.attn_k.weight", "offset": 1656963072, "shape": [ 6144, 1024 ], "size": 4325376, "type": 11 }, { "name": "blk.12.attn_q.weight", "offset": 1659666432, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.12.attn_v.weight", "offset": 1675886592, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 1680211968, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1680236544, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 1749442560, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 1749467136, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1749491712, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 1818697728, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 1861951488, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 1905205248, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1905229824, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 1907933184, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 1933885440, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 1950105600, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 1954430976, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1954455552, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 2023661568, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 2066915328, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 2110169088, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2110193664, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 2112897024, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 2138849280, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "\u0001\u0000\u0000\u0000\u0000\u0018\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\ufffd", "offset": 2155069440, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 2159394816, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2159419392, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 2228625408, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 2271879168, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 2315132928, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2315157504, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 2317860864, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 2343813120, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 2360033280, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 2364358656, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2364383232, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 2433589248, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 2476843008, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 2520096768, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2520121344, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 2522824704, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 2548776960, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.9.attn_v.weight", "offset": 2564997120, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 2569322496, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2569347072, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 2638553088, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 2681806848, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 2725060608, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2725085184, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 2751037440, "shape": [ 6144 ], "size": 25952256, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2751062016, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 2820268032, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 2863521792, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 2906775552, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2906800128, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 2909503488, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 2935455744, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.13.attn_v.weight", "offset": 2951675904, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 2956001280, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2956025856, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 3025231872, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 3068485632, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 3111739392, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 3111763968, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 3114467328, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 3140419584, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 3156639744, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 3160965120, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3160989696, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 3230195712, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 3273449472, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 3316703232, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 3316727808, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 3319431168, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 3345383424, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 3361603584, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 3365928960, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3365953536, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 3435159552, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 3478413312, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 3521667072, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 3521691648, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 3524395008, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 3550347264, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 3566567424, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 3570892800, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3570917376, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 3640123392, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 3683377152, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 3726630912, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 3726655488, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 3729358848, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 3755311104, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.17.attn_v.weight", "offset": 3771531264, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 3775856640, "shape": [ 6144, 16384 ], "size": 4325376, "type": 11 }, { "name": "blk.18.attn_k.weight", "offset": 3819110400, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 3821813760, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 3847766016, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 3863986176, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 3868311552, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3868336128, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 3937542144, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 3980795904, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 3980820480, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3980845056, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 4050051072, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 4093304832, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 4136558592, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 4136583168, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 4139286528, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 4165238784, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 4181458944, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 4185784320, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4185808896, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 4255014912, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 4298268672, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 4341522432, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 4341547008, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 4344250368, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 4370202624, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 4386422784, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 4390748160, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4390772736, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.21.ffn_gate.weight", "offset": 4459978752, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 4503232512, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 4546486272, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 4546510848, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 4549214208, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 4575166464, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 4591386624, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 4595712000, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4595736576, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 4664942592, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 4708196352, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 4751450112, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 4751474688, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 4754178048, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 4780130304, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 4796350464, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 4800675840, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4800700416, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 4869906432, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 4913160192, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 4956413952, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 4956438528, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 4959141888, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.23.attn_q.weight", "offset": 4985094144, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 5001314304, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.24.ffn_gate.weight", "offset": 5005639680, "shape": [ 6144, 16384 ], "size": 4325376, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 5048893440, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.24.attn_k.weight", "offset": 5092147200, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 5094850560, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.24.attn_q.weight", "offset": 5120802816, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 5137022976, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 5141348352, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5141372928, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 5210578944, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.25.attn_norm.weight", "offset": 5210603520, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5210628096, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 5279834112, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 5323087872, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 5366341632, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 5366366208, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 5369069568, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 5395021824, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 5411241984, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 5415567360, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5415591936, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 5484797952, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 5528051712, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 5571305472, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 5571330048, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 5574033408, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 5599985664, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 5616205824, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 5620531200, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5620555776, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 5689761792, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 5733015552, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 5776269312, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 5776293888, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 5778997248, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.27.attn_q.weight", "offset": 5804949504, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 5821169664, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 5825495040, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5825519616, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.28.ffn_gate.weight", "offset": 5894725632, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 5937979392, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 5981233152, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 5981257728, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 5983961088, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 6009913344, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 6026133504, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 6030458880, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 6030483456, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.29.ffn_gate.weight", "offset": 6099689472, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 6142943232, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 6186196992, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 6186221568, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 6188924928, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 6214877184, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 6231097344, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 6235422720, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 6235447296, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.30.ffn_gate.weight", "offset": 6304653312, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 6347907072, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 6391160832, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 6391185408, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 6393888768, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 6419841024, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 6436061184, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.31.attn_k.weight", "offset": 6440386560, "shape": [ 6144, 1024 ], "size": 4325376, "type": 11 }, { "name": "blk.31.attn_q.weight", "offset": 6443089920, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 6459310080, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 6463635456, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 6463660032, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.31.ffn_gate.weight", "offset": 6532866048, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 6576119808, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 6619373568, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 6619398144, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 6645350400, "shape": [ 6144 ], "size": 25952256, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 6645374976, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.32.ffn_gate.weight", "offset": 6714580992, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 6757834752, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 6801088512, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 6801113088, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 6803816448, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.32.attn_q.weight", "offset": 6829768704, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.32.attn_v.weight", "offset": 6845988864, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.33.attn_norm.weight", "offset": 6850314240, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6850338816, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.33.ffn_gate.weight", "offset": 6919544832, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 6962798592, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 7006052352, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 7006076928, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 7008780288, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.33.attn_q.weight", "offset": 7034732544, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.33.attn_v.weight", "offset": 7050952704, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 7055278080, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 7055302656, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.34.ffn_gate.weight", "offset": 7124508672, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 7167762432, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 7211016192, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 7211040768, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 7213744128, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.34.attn_q.weight", "offset": 7239696384, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.34.attn_v.weight", "offset": 7255916544, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.35.attn_norm.weight", "offset": 7260241920, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 7260266496, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.35.ffn_gate.weight", "offset": 7329472512, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 7372726272, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 7415980032, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 7416004608, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 7418707968, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.35.attn_q.weight", "offset": 7444660224, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.35.attn_v.weight", "offset": 7460880384, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 7465205760, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 7465230336, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 7534436352, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 7577690112, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 7620943872, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 7620968448, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 7623671808, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.36.attn_q.weight", "offset": 7649624064, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.36.attn_v.weight", "offset": 7665844224, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.37.ffn_gate.weight", "offset": 7670169600, "shape": [ 6144, 16384 ], "size": 4325376, "type": 11 }, { "name": "blk.37.attn_k.weight", "offset": 7713423360, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 7716126720, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.37.attn_q.weight", "offset": 7742078976, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.37.attn_v.weight", "offset": 7758299136, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.37.attn_norm.weight", "offset": 7762624512, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 7762649088, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.37.ffn_up.weight", "offset": 7831855104, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 7875108864, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.38.attn_norm.weight", "offset": 7875133440, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 7875158016, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.38.ffn_gate.weight", "offset": 7944364032, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 7987617792, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 8030871552, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 8030896128, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 8033599488, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.38.attn_q.weight", "offset": 8059551744, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.38.attn_v.weight", "offset": 8075771904, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 8080097280, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 8080121856, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.39.ffn_gate.weight", "offset": 8149327872, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 8192581632, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 8235835392, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 8235859968, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 8238563328, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.39.attn_q.weight", "offset": 8264515584, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.39.attn_v.weight", "offset": 8280735744, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 8285061120, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 8285085696, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.40.ffn_gate.weight", "offset": 8354291712, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.40.ffn_up.weight", "offset": 8397545472, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.40.ffn_norm.weight", "offset": 8440799232, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 8440823808, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 8443527168, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.40.attn_q.weight", "offset": 8469479424, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.40.attn_v.weight", "offset": 8485699584, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 8490024960, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 8490049536, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.41.ffn_gate.weight", "offset": 8559255552, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.41.ffn_up.weight", "offset": 8602509312, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.41.ffn_norm.weight", "offset": 8645763072, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 8645787648, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 8648491008, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.41.attn_q.weight", "offset": 8674443264, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.41.attn_v.weight", "offset": 8690663424, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.42.attn_norm.weight", "offset": 8694988800, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 8695013376, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.42.ffn_gate.weight", "offset": 8764219392, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.42.ffn_up.weight", "offset": 8807473152, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.42.ffn_norm.weight", "offset": 8850726912, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 8850751488, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.42.attn_output.weight", "offset": 8853454848, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.42.attn_q.weight", "offset": 8879407104, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.42.attn_v.weight", "offset": 8895627264, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.43.ffn_gate.weight", "offset": 8899952640, "shape": [ 6144, 16384 ], "size": 4325376, "type": 11 }, { "name": "blk.43.ffn_up.weight", "offset": 8943206400, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.43.attn_k.weight", "offset": 8986460160, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.43.attn_output.weight", "offset": 8989163520, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.43.attn_q.weight", "offset": 9015115776, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.43.attn_v.weight", "offset": 9031335936, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 9035661312, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 9035685888, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.43.ffn_norm.weight", "offset": 9104891904, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 9104916480, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 9104941056, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.44.ffn_gate.weight", "offset": 9174147072, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.44.ffn_up.weight", "offset": 9217400832, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.44.ffn_norm.weight", "offset": 9260654592, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 9260679168, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.44.attn_output.weight", "offset": 9263382528, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.44.attn_q.weight", "offset": 9289334784, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.44.attn_v.weight", "offset": 9305554944, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 9309880320, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 9309904896, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.45.ffn_gate.weight", "offset": 9379110912, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.45.ffn_up.weight", "offset": 9422364672, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.45.ffn_norm.weight", "offset": 9465618432, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 9465643008, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.45.attn_output.weight", "offset": 9468346368, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.45.attn_q.weight", "offset": 9494298624, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.45.attn_v.weight", "offset": 9510518784, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.46.attn_norm.weight", "offset": 9514844160, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 9514868736, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.46.ffn_gate.weight", "offset": 9584074752, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.46.ffn_up.weight", "offset": 9627328512, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.46.ffn_norm.weight", "offset": 9670582272, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 9670606848, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.46.attn_output.weight", "offset": 9673310208, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.46.attn_q.weight", "offset": 9699262464, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.46.attn_v.weight", "offset": 9715482624, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 9719808000, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 9719832576, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.47.ffn_gate.weight", "offset": 9789038592, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.47.ffn_up.weight", "offset": 9832292352, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.47.ffn_norm.weight", "offset": 9875546112, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 9875570688, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.47.attn_output.weight", "offset": 9878274048, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.47.attn_q.weight", "offset": 9904226304, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.47.attn_v.weight", "offset": 9920446464, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.48.attn_norm.weight", "offset": 9924771840, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 9924796416, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.48.ffn_gate.weight", "offset": 9994002432, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.48.ffn_up.weight", "offset": 10037256192, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.48.ffn_norm.weight", "offset": 10080509952, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 10080534528, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.48.attn_output.weight", "offset": 10083237888, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.48.attn_q.weight", "offset": 10109190144, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.48.attn_v.weight", "offset": 10125410304, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.49.attn_norm.weight", "offset": 10129735680, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 10129760256, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.49.ffn_gate.weight", "offset": 10198966272, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.49.ffn_up.weight", "offset": 10242220032, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.49.ffn_norm.weight", "offset": 10285473792, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 10285498368, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.49.attn_output.weight", "offset": 10288201728, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.49.attn_q.weight", "offset": 10314153984, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.49.attn_v.weight", "offset": 10330374144, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.50.attn_k.weight", "offset": 10334699520, "shape": [ 6144, 1024 ], "size": 4325376, "type": 11 }, { "name": "blk.50.attn_q.weight", "offset": 10337402880, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.50.attn_v.weight", "offset": 10353623040, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "output.weight", "offset": 10357948416, "shape": [ 6144, 32768 ], "size": 4325376, "type": 14 }, { "name": "blk.50.attn_norm.weight", "offset": 10523099136, "shape": [ 6144 ], "size": 165150720, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 10523123712, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.50.ffn_gate.weight", "offset": 10592329728, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.50.ffn_up.weight", "offset": 10635583488, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.50.ffn_norm.weight", "offset": 10678837248, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 10678861824, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.51.attn_norm.weight", "offset": 10704814080, "shape": [ 6144 ], "size": 25952256, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 10704838656, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.51.ffn_gate.weight", "offset": 10774044672, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.51.ffn_up.weight", "offset": 10817298432, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.51.ffn_norm.weight", "offset": 10860552192, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 10860576768, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.51.attn_output.weight", "offset": 10863280128, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.51.attn_q.weight", "offset": 10889232384, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.51.attn_v.weight", "offset": 10905452544, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.52.attn_norm.weight", "offset": 10909777920, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 10909802496, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.52.ffn_gate.weight", "offset": 10979008512, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.52.ffn_up.weight", "offset": 11022262272, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.52.ffn_norm.weight", "offset": 11065516032, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 11065540608, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.52.attn_output.weight", "offset": 11068243968, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.52.attn_q.weight", "offset": 11094196224, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.52.attn_v.weight", "offset": 11110416384, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.53.attn_norm.weight", "offset": 11114741760, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 11114766336, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.53.ffn_gate.weight", "offset": 11183972352, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.53.ffn_up.weight", "offset": 11227226112, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.53.ffn_norm.weight", "offset": 11270479872, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 11270504448, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.53.attn_output.weight", "offset": 11273207808, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.53.attn_q.weight", "offset": 11299160064, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.53.attn_v.weight", "offset": 11315380224, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.54.attn_norm.weight", "offset": 11319705600, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 11319730176, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.54.ffn_gate.weight", "offset": 11388936192, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.54.ffn_up.weight", "offset": 11432189952, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.54.ffn_norm.weight", "offset": 11475443712, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 11475468288, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.54.attn_output.weight", "offset": 11478171648, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.54.attn_q.weight", "offset": 11504123904, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.54.attn_v.weight", "offset": 11520344064, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "blk.55.attn_norm.weight", "offset": 11524669440, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 11524694016, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.55.ffn_gate.weight", "offset": 11593900032, "shape": [ 6144, 16384 ], "size": 69206016, "type": 11 }, { "name": "blk.55.ffn_up.weight", "offset": 11637153792, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.55.ffn_norm.weight", "offset": 11680407552, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 11680432128, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.55.attn_output.weight", "offset": 11683135488, "shape": [ 6144, 6144 ], "size": 2703360, "type": 13 }, { "name": "blk.55.attn_q.weight", "offset": 11709087744, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.55.attn_v.weight", "offset": 11725307904, "shape": [ 6144, 1024 ], "size": 16220160, "type": 13 }, { "name": "output_norm.weight", "offset": 11729633280, "shape": [ 6144 ], "size": 4325376, "type": 0 } ], "version": 3 }