codestral

{ "metadata": { "general.architecture": "llama", "general.file_type": 10, "general.name": "Codestral-22B-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 32768, "llama.embedding_length": 6144, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_space_prefix": true, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 22247282688, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 66060288, "shape": [ 6144 ], "size": 66060288, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 66084864, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 109338624, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.0.ffn_up.weight", "offset": 142368768, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.0.ffn_norm.weight", "offset": 175398912, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 175423488, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.0.attn_output.weight", "offset": 177487872, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.0.attn_q.weight", "offset": 193708032, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.0.attn_v.weight", "offset": 206094336, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.1.attn_norm.weight", "offset": 209633280, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 209657856, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.1.ffn_gate.weight", "offset": 252911616, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.1.ffn_up.weight", "offset": 285941760, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.1.ffn_norm.weight", "offset": 318971904, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 318996480, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.1.attn_output.weight", "offset": 321060864, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.1.attn_q.weight", "offset": 337281024, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.1.attn_v.weight", "offset": 349667328, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 353206272, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 353230848, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.2.ffn_gate.weight", "offset": 396484608, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.2.ffn_up.weight", "offset": 429514752, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.2.ffn_norm.weight", "offset": 462544896, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 462569472, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.2.attn_output.weight", "offset": 464633856, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.2.attn_q.weight", "offset": 480854016, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.2.attn_v.weight", "offset": 493240320, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 496779264, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 496803840, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.3.ffn_gate.weight", "offset": 540057600, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.3.ffn_up.weight", "offset": 573087744, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.3.ffn_norm.weight", "offset": 606117888, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 606142464, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.3.attn_output.weight", "offset": 608206848, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.3.attn_q.weight", "offset": 624427008, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.3.attn_v.weight", "offset": 636813312, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 640352256, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 640376832, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.4.ffn_gate.weight", "offset": 683630592, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.4.ffn_up.weight", "offset": 716660736, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.4.ffn_norm.weight", "offset": 749690880, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 749715456, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.4.attn_output.weight", "offset": 751779840, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.4.attn_q.weight", "offset": 768000000, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.4.attn_v.weight", "offset": 780386304, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 783925248, "shape": [ 6144, 16384 ], "size": 3538944, "type": 10 }, { "name": "blk.5.ffn_up.weight", "offset": 816955392, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.5.attn_k.weight", "offset": 849985536, "shape": [ 6144, 1024 ], "size": 33030144, "type": 10 }, { "name": "blk.5.attn_output.weight", "offset": 852049920, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.5.attn_q.weight", "offset": 868270080, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.5.attn_v.weight", "offset": 880656384, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 884195328, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 884219904, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.10.ffn_gate.weight", "offset": 927473664, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.10.ffn_up.weight", "offset": 960503808, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.10.ffn_norm.weight", "offset": 993533952, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 993558528, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.10.attn_output.weight", "offset": 995622912, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.10.attn_q.weight", "offset": 1011843072, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.10.attn_v.weight", "offset": 1024229376, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1027768320, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1027792896, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.11.ffn_gate.weight", "offset": 1071046656, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.11.ffn_up.weight", "offset": 1104076800, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.11.ffn_norm.weight", "offset": 1137106944, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1137131520, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.11.attn_output.weight", "offset": 1139195904, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.11.attn_q.weight", "offset": 1155416064, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.11.attn_v.weight", "offset": 1167802368, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.12.attn_k.weight", "offset": 1171341312, "shape": [ 6144, 1024 ], "size": 3538944, "type": 10 }, { "name": "blk.12.attn_q.weight", "offset": 1173405696, "shape": [ 6144, 6144 ], "size": 2064384, "type": 10 }, { "name": "blk.12.attn_v.weight", "offset": 1185792000, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 1189330944, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1189355520, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 1232609280, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 1232633856, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1232658432, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.6.ffn_gate.weight", "offset": 1275912192, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.6.ffn_up.weight", "offset": 1308942336, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.6.ffn_norm.weight", "offset": 1341972480, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1341997056, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.6.attn_output.weight", "offset": 1344061440, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.6.attn_q.weight", "offset": 1360281600, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.6.attn_v.weight", "offset": 1372667904, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 1376206848, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1376231424, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.7.ffn_gate.weight", "offset": 1419485184, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.7.ffn_up.weight", "offset": 1452515328, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.7.ffn_norm.weight", "offset": 1485545472, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1485570048, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.7.attn_output.weight", "offset": 1487634432, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.7.attn_q.weight", "offset": 1503854592, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "\u0001\u0000\u0000\u0000\u0000\u0018\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 e", "offset": 1516240896, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 1519779840, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1519804416, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.8.ffn_gate.weight", "offset": 1563058176, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.8.ffn_up.weight", "offset": 1596088320, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.8.ffn_norm.weight", "offset": 1629118464, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1629143040, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.8.attn_output.weight", "offset": 1631207424, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.8.attn_q.weight", "offset": 1647427584, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.8.attn_v.weight", "offset": 1659813888, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 1663352832, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1663377408, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.9.ffn_gate.weight", "offset": 1706631168, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.9.ffn_up.weight", "offset": 1739661312, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.9.ffn_norm.weight", "offset": 1772691456, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1772716032, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.9.attn_output.weight", "offset": 1774780416, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.9.attn_q.weight", "offset": 1791000576, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.9.attn_v.weight", "offset": 1803386880, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1806925824, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1806950400, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 1850204160, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.12.ffn_up.weight", "offset": 1883234304, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.12.ffn_norm.weight", "offset": 1916264448, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1916289024, "shape": [ 6144, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 1932509184, "shape": [ 6144 ], "size": 16220160, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1932533760, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 1975787520, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.13.ffn_up.weight", "offset": 2008817664, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.13.ffn_norm.weight", "offset": 2041847808, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2041872384, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.13.attn_output.weight", "offset": 2043936768, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.13.attn_q.weight", "offset": 2060156928, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.13.attn_v.weight", "offset": 2072543232, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 2076082176, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2076106752, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.14.ffn_gate.weight", "offset": 2119360512, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.14.ffn_up.weight", "offset": 2152390656, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.14.ffn_norm.weight", "offset": 2185420800, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2185445376, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.14.attn_output.weight", "offset": 2187509760, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.14.attn_q.weight", "offset": 2203729920, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.14.attn_v.weight", "offset": 2216116224, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 2219655168, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2219679744, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.15.ffn_gate.weight", "offset": 2262933504, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.15.ffn_up.weight", "offset": 2295963648, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.15.ffn_norm.weight", "offset": 2328993792, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2329018368, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.15.attn_output.weight", "offset": 2331082752, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.15.attn_q.weight", "offset": 2347302912, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.15.attn_v.weight", "offset": 2359689216, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 2363228160, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2363252736, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 2406506496, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.16.ffn_up.weight", "offset": 2439536640, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.16.ffn_norm.weight", "offset": 2472566784, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2472591360, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.16.attn_output.weight", "offset": 2474655744, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.16.attn_q.weight", "offset": 2490875904, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.16.attn_v.weight", "offset": 2503262208, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 2506801152, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2506825728, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 2550079488, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.17.ffn_up.weight", "offset": 2583109632, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.17.ffn_norm.weight", "offset": 2616139776, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2616164352, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.17.attn_output.weight", "offset": 2618228736, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.17.attn_q.weight", "offset": 2634448896, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.17.attn_v.weight", "offset": 2646835200, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 2650374144, "shape": [ 6144, 16384 ], "size": 3538944, "type": 10 }, { "name": "blk.18.attn_k.weight", "offset": 2683404288, "shape": [ 6144, 1024 ], "size": 33030144, "type": 10 }, { "name": "blk.18.attn_output.weight", "offset": 2685468672, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.18.attn_q.weight", "offset": 2701688832, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.18.attn_v.weight", "offset": 2714075136, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 2717614080, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2717638656, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 2760892416, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.18.ffn_norm.weight", "offset": 2793922560, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 2793947136, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2793971712, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.19.ffn_gate.weight", "offset": 2837225472, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.19.ffn_up.weight", "offset": 2870255616, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.19.ffn_norm.weight", "offset": 2903285760, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2903310336, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.19.attn_output.weight", "offset": 2905374720, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.19.attn_q.weight", "offset": 2921594880, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.19.attn_v.weight", "offset": 2933981184, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 2937520128, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2937544704, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.20.ffn_gate.weight", "offset": 2980798464, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.20.ffn_up.weight", "offset": 3013828608, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.20.ffn_norm.weight", "offset": 3046858752, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 3046883328, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.20.attn_output.weight", "offset": 3048947712, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.20.attn_q.weight", "offset": 3065167872, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.20.attn_v.weight", "offset": 3077554176, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 3081093120, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3081117696, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.21.ffn_gate.weight", "offset": 3124371456, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.21.ffn_up.weight", "offset": 3157401600, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.21.ffn_norm.weight", "offset": 3190431744, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 3190456320, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.21.attn_output.weight", "offset": 3192520704, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.21.attn_q.weight", "offset": 3208740864, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.21.attn_v.weight", "offset": 3221127168, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 3224666112, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3224690688, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.22.ffn_gate.weight", "offset": 3267944448, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.22.ffn_up.weight", "offset": 3300974592, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.22.ffn_norm.weight", "offset": 3334004736, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3334029312, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.22.attn_output.weight", "offset": 3336093696, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.22.attn_q.weight", "offset": 3352313856, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.22.attn_v.weight", "offset": 3364700160, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 3368239104, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3368263680, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.23.ffn_gate.weight", "offset": 3411517440, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.23.ffn_up.weight", "offset": 3444547584, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.23.ffn_norm.weight", "offset": 3477577728, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3477602304, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.23.attn_output.weight", "offset": 3479666688, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.23.attn_q.weight", "offset": 3495886848, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.23.attn_v.weight", "offset": 3508273152, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 3511812096, "shape": [ 6144, 16384 ], "size": 3538944, "type": 10 }, { "name": "blk.24.ffn_up.weight", "offset": 3544842240, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.24.attn_k.weight", "offset": 3577872384, "shape": [ 6144, 1024 ], "size": 33030144, "type": 10 }, { "name": "blk.24.attn_output.weight", "offset": 3579936768, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.24.attn_q.weight", "offset": 3596156928, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.24.attn_v.weight", "offset": 3608543232, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 3612082176, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3612106752, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 3655360512, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.25.attn_norm.weight", "offset": 3655385088, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3655409664, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.25.ffn_gate.weight", "offset": 3698663424, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.25.ffn_up.weight", "offset": 3731693568, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.25.ffn_norm.weight", "offset": 3764723712, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3764748288, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.25.attn_output.weight", "offset": 3766812672, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.25.attn_q.weight", "offset": 3783032832, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.25.attn_v.weight", "offset": 3795419136, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 3798958080, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3798982656, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.26.ffn_gate.weight", "offset": 3842236416, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.26.ffn_up.weight", "offset": 3875266560, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.26.ffn_norm.weight", "offset": 3908296704, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3908321280, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.26.attn_output.weight", "offset": 3910385664, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.26.attn_q.weight", "offset": 3926605824, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.26.attn_v.weight", "offset": 3938992128, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 3942531072, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3942555648, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.27.ffn_gate.weight", "offset": 3985809408, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.27.ffn_up.weight", "offset": 4018839552, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.27.ffn_norm.weight", "offset": 4051869696, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 4051894272, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.27.attn_output.weight", "offset": 4053958656, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.27.attn_q.weight", "offset": 4070178816, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.27.attn_v.weight", "offset": 4082565120, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 4086104064, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 4086128640, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.28.ffn_gate.weight", "offset": 4129382400, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.28.ffn_up.weight", "offset": 4162412544, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.28.ffn_norm.weight", "offset": 4195442688, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 4195467264, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.28.attn_output.weight", "offset": 4197531648, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.28.attn_q.weight", "offset": 4213751808, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.28.attn_v.weight", "offset": 4226138112, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 4229677056, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 4229701632, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.29.ffn_gate.weight", "offset": 4272955392, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.29.ffn_up.weight", "offset": 4305985536, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.29.ffn_norm.weight", "offset": 4339015680, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 4339040256, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.29.attn_output.weight", "offset": 4341104640, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.29.attn_q.weight", "offset": 4357324800, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.29.attn_v.weight", "offset": 4369711104, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 4373250048, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 4373274624, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.30.ffn_gate.weight", "offset": 4416528384, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.30.ffn_up.weight", "offset": 4449558528, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.30.ffn_norm.weight", "offset": 4482588672, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 4482613248, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.30.attn_output.weight", "offset": 4484677632, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.30.attn_q.weight", "offset": 4500897792, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.30.attn_v.weight", "offset": 4513284096, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.31.attn_k.weight", "offset": 4516823040, "shape": [ 6144, 1024 ], "size": 3538944, "type": 10 }, { "name": "blk.31.attn_q.weight", "offset": 4518887424, "shape": [ 6144, 6144 ], "size": 2064384, "type": 10 }, { "name": "blk.31.attn_v.weight", "offset": 4531273728, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 4534812672, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 4534837248, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.31.ffn_gate.weight", "offset": 4578091008, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.31.ffn_up.weight", "offset": 4611121152, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.31.ffn_norm.weight", "offset": 4644151296, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 4644175872, "shape": [ 6144, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.32.attn_norm.weight", "offset": 4660396032, "shape": [ 6144 ], "size": 16220160, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 4660420608, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.32.ffn_gate.weight", "offset": 4703674368, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.32.ffn_up.weight", "offset": 4736704512, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.32.ffn_norm.weight", "offset": 4769734656, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 4769759232, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.32.attn_output.weight", "offset": 4771823616, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.32.attn_q.weight", "offset": 4788043776, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.32.attn_v.weight", "offset": 4800430080, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 4803969024, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 4803993600, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.33.ffn_gate.weight", "offset": 4847247360, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.33.ffn_up.weight", "offset": 4880277504, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.33.ffn_norm.weight", "offset": 4913307648, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 4913332224, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.33.attn_output.weight", "offset": 4915396608, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.33.attn_q.weight", "offset": 4931616768, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.33.attn_v.weight", "offset": 4944003072, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 4947542016, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 4947566592, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.34.ffn_gate.weight", "offset": 4990820352, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.34.ffn_up.weight", "offset": 5023850496, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.34.ffn_norm.weight", "offset": 5056880640, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 5056905216, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.34.attn_output.weight", "offset": 5058969600, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.34.attn_q.weight", "offset": 5075189760, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.34.attn_v.weight", "offset": 5087576064, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.35.attn_norm.weight", "offset": 5091115008, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 5091139584, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.35.ffn_gate.weight", "offset": 5134393344, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.35.ffn_up.weight", "offset": 5167423488, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.35.ffn_norm.weight", "offset": 5200453632, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 5200478208, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.35.attn_output.weight", "offset": 5202542592, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.35.attn_q.weight", "offset": 5218762752, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.35.attn_v.weight", "offset": 5231149056, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.36.attn_norm.weight", "offset": 5234688000, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 5234712576, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.36.ffn_gate.weight", "offset": 5277966336, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.36.ffn_up.weight", "offset": 5310996480, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.36.ffn_norm.weight", "offset": 5344026624, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 5344051200, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.36.attn_output.weight", "offset": 5346115584, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.36.attn_q.weight", "offset": 5362335744, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.36.attn_v.weight", "offset": 5374722048, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.37.ffn_gate.weight", "offset": 5378260992, "shape": [ 6144, 16384 ], "size": 3538944, "type": 10 }, { "name": "blk.37.attn_k.weight", "offset": 5411291136, "shape": [ 6144, 1024 ], "size": 33030144, "type": 10 }, { "name": "blk.37.attn_output.weight", "offset": 5413355520, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.37.attn_q.weight", "offset": 5429575680, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.37.attn_v.weight", "offset": 5441961984, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 5445500928, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 5445525504, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 5488779264, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.37.ffn_norm.weight", "offset": 5521809408, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.38.attn_norm.weight", "offset": 5521833984, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 5521858560, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.38.ffn_gate.weight", "offset": 5565112320, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.38.ffn_up.weight", "offset": 5598142464, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.38.ffn_norm.weight", "offset": 5631172608, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 5631197184, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.38.attn_output.weight", "offset": 5633261568, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.38.attn_q.weight", "offset": 5649481728, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.38.attn_v.weight", "offset": 5661868032, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 5665406976, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 5665431552, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.39.ffn_gate.weight", "offset": 5708685312, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.39.ffn_up.weight", "offset": 5741715456, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.39.ffn_norm.weight", "offset": 5774745600, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 5774770176, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.39.attn_output.weight", "offset": 5776834560, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.39.attn_q.weight", "offset": 5793054720, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.39.attn_v.weight", "offset": 5805441024, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.40.attn_norm.weight", "offset": 5808979968, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 5809004544, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.40.ffn_gate.weight", "offset": 5852258304, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.40.ffn_up.weight", "offset": 5885288448, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.40.ffn_norm.weight", "offset": 5918318592, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 5918343168, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "\u0000blk.55.attn_q.weight\u0002\u0000\u0000\u0000", "offset": 5920407552, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.40.attn_q.weight", "offset": 5936627712, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.40.attn_v.weight", "offset": 5949014016, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.41.attn_norm.weight", "offset": 5952552960, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 5952577536, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.41.ffn_gate.weight", "offset": 5995831296, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.41.ffn_up.weight", "offset": 6028861440, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.41.ffn_norm.weight", "offset": 6061891584, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 6061916160, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.41.attn_output.weight", "offset": 6063980544, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.41.attn_q.weight", "offset": 6080200704, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.41.attn_v.weight", "offset": 6092587008, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.42.attn_norm.weight", "offset": 6096125952, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 6096150528, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.42.ffn_gate.weight", "offset": 6139404288, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.42.ffn_up.weight", "offset": 6172434432, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.42.ffn_norm.weight", "offset": 6205464576, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 6205489152, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.42.attn_output.weight", "offset": 6207553536, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.42.attn_q.weight", "offset": 6223773696, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.42.attn_v.weight", "offset": 6236160000, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.43.ffn_gate.weight", "offset": 6239698944, "shape": [ 6144, 16384 ], "size": 3538944, "type": 10 }, { "name": "blk.43.ffn_up.weight", "offset": 6272729088, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.43.attn_k.weight", "offset": 6305759232, "shape": [ 6144, 1024 ], "size": 33030144, "type": 10 }, { "name": "blk.43.attn_output.weight", "offset": 6307823616, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.43.attn_q.weight", "offset": 6324043776, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.43.attn_v.weight", "offset": 6336430080, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.43.attn_norm.weight", "offset": 6339969024, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 6339993600, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.43.ffn_norm.weight", "offset": 6383247360, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 6383271936, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 6383296512, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.44.ffn_gate.weight", "offset": 6426550272, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.44.ffn_up.weight", "offset": 6459580416, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.44.ffn_norm.weight", "offset": 6492610560, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 6492635136, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.44.attn_output.weight", "offset": 6494699520, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.44.attn_q.weight", "offset": 6510919680, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.44.attn_v.weight", "offset": 6523305984, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.45.attn_norm.weight", "offset": 6526844928, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 6526869504, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.45.ffn_gate.weight", "offset": 6570123264, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.45.ffn_up.weight", "offset": 6603153408, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.45.ffn_norm.weight", "offset": 6636183552, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 6636208128, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.45.attn_output.weight", "offset": 6638272512, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.45.attn_q.weight", "offset": 6654492672, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.45.attn_v.weight", "offset": 6666878976, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.46.attn_norm.weight", "offset": 6670417920, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 6670442496, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.46.ffn_gate.weight", "offset": 6713696256, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.46.ffn_up.weight", "offset": 6746726400, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.46.ffn_norm.weight", "offset": 6779756544, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 6779781120, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.46.attn_output.weight", "offset": 6781845504, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.46.attn_q.weight", "offset": 6798065664, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.46.attn_v.weight", "offset": 6810451968, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.47.attn_norm.weight", "offset": 6813990912, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 6814015488, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.47.ffn_gate.weight", "offset": 6857269248, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.47.ffn_up.weight", "offset": 6890299392, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.47.ffn_norm.weight", "offset": 6923329536, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 6923354112, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.47.attn_output.weight", "offset": 6925418496, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.47.attn_q.weight", "offset": 6941638656, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.47.attn_v.weight", "offset": 6954024960, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.48.attn_norm.weight", "offset": 6957563904, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 6957588480, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.48.ffn_gate.weight", "offset": 7000842240, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.48.ffn_up.weight", "offset": 7033872384, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.48.ffn_norm.weight", "offset": 7066902528, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 7066927104, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.48.attn_output.weight", "offset": 7068991488, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.48.attn_q.weight", "offset": 7085211648, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.48.attn_v.weight", "offset": 7097597952, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.49.attn_norm.weight", "offset": 7101136896, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 7101161472, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.49.ffn_gate.weight", "offset": 7144415232, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.49.ffn_up.weight", "offset": 7177445376, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.49.ffn_norm.weight", "offset": 7210475520, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 7210500096, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.49.attn_output.weight", "offset": 7212564480, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.49.attn_q.weight", "offset": 7228784640, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.49.attn_v.weight", "offset": 7241170944, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.50.attn_k.weight", "offset": 7244709888, "shape": [ 6144, 1024 ], "size": 3538944, "type": 10 }, { "name": "blk.50.attn_q.weight", "offset": 7246774272, "shape": [ 6144, 6144 ], "size": 2064384, "type": 10 }, { "name": "blk.50.attn_v.weight", "offset": 7259160576, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "output.weight", "offset": 7262699520, "shape": [ 6144, 32768 ], "size": 3538944, "type": 14 }, { "name": "blk.50.attn_norm.weight", "offset": 7427850240, "shape": [ 6144 ], "size": 165150720, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 7427874816, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.50.ffn_gate.weight", "offset": 7471128576, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.50.ffn_up.weight", "offset": 7504158720, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.50.ffn_norm.weight", "offset": 7537188864, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 7537213440, "shape": [ 6144, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.51.attn_norm.weight", "offset": 7553433600, "shape": [ 6144 ], "size": 16220160, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 7553458176, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.51.ffn_gate.weight", "offset": 7596711936, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.51.ffn_up.weight", "offset": 7629742080, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.51.ffn_norm.weight", "offset": 7662772224, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 7662796800, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.51.attn_output.weight", "offset": 7664861184, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.51.attn_q.weight", "offset": 7681081344, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.51.attn_v.weight", "offset": 7693467648, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.52.attn_norm.weight", "offset": 7697006592, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 7697031168, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.52.ffn_gate.weight", "offset": 7740284928, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.52.ffn_up.weight", "offset": 7773315072, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.52.ffn_norm.weight", "offset": 7806345216, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 7806369792, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.52.attn_output.weight", "offset": 7808434176, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.52.attn_q.weight", "offset": 7824654336, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.52.attn_v.weight", "offset": 7837040640, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.53.attn_norm.weight", "offset": 7840579584, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 7840604160, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.53.ffn_gate.weight", "offset": 7883857920, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.53.ffn_up.weight", "offset": 7916888064, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.53.ffn_norm.weight", "offset": 7949918208, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 7949942784, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.53.attn_output.weight", "offset": 7952007168, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.53.attn_q.weight", "offset": 7968227328, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.53.attn_v.weight", "offset": 7980613632, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.54.attn_norm.weight", "offset": 7984152576, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 7984177152, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.54.ffn_gate.weight", "offset": 8027430912, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.54.ffn_up.weight", "offset": 8060461056, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.54.ffn_norm.weight", "offset": 8093491200, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 8093515776, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.54.attn_output.weight", "offset": 8095580160, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.54.attn_q.weight", "offset": 8111800320, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.54.attn_v.weight", "offset": 8124186624, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "blk.55.attn_norm.weight", "offset": 8127725568, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 8127750144, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.55.ffn_gate.weight", "offset": 8171003904, "shape": [ 6144, 16384 ], "size": 43253760, "type": 10 }, { "name": "blk.55.ffn_up.weight", "offset": 8204034048, "shape": [ 6144, 16384 ], "size": 33030144, "type": 10 }, { "name": "blk.55.ffn_norm.weight", "offset": 8237064192, "shape": [ 6144 ], "size": 33030144, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 8237088768, "shape": [ 6144, 1024 ], "size": 24576, "type": 10 }, { "name": "blk.55.attn_output.weight", "offset": 8239153152, "shape": [ 6144, 6144 ], "size": 2064384, "type": 11 }, { "name": "blk.55.attn_q.weight", "offset": 8255373312, "shape": [ 6144, 6144 ], "size": 16220160, "type": 10 }, { "name": "blk.55.attn_v.weight", "offset": 8267759616, "shape": [ 6144, 1024 ], "size": 12386304, "type": 12 }, { "name": "output_norm.weight", "offset": 8271298560, "shape": [ 6144 ], "size": 3538944, "type": 0 } ], "version": 3 }