codestral

{ "metadata": { "general.architecture": "llama", "general.file_type": 2, "general.name": "Codestral-22B-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 32768, "llama.embedding_length": 6144, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_space_prefix": true, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 22247282688, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 113246208, "shape": [ 6144 ], "size": 113246208, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 113270784, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.0.ffn_gate.weight", "offset": 169893888, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 226516992, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 283140096, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 283164672, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.0.attn_output.weight", "offset": 286703616, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.0.attn_q.weight", "offset": 307937280, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.0.attn_v.weight", "offset": 329170944, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 332709888, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 332734464, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.1.ffn_gate.weight", "offset": 389357568, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 445980672, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 502603776, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 502628352, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.1.attn_output.weight", "offset": 506167296, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.1.attn_q.weight", "offset": 527400960, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.1.attn_v.weight", "offset": 548634624, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 552173568, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 552198144, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.2.ffn_gate.weight", "offset": 608821248, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 665444352, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 722067456, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 722092032, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.2.attn_output.weight", "offset": 725630976, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.2.attn_q.weight", "offset": 746864640, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.2.attn_v.weight", "offset": 768098304, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 771637248, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 771661824, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.3.ffn_gate.weight", "offset": 828284928, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 884908032, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 941531136, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 941555712, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.3.attn_output.weight", "offset": 945094656, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.3.attn_q.weight", "offset": 966328320, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.3.attn_v.weight", "offset": 987561984, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 991100928, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 991125504, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.4.ffn_gate.weight", "offset": 1047748608, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 1104371712, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 1160994816, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1161019392, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.4.attn_output.weight", "offset": 1164558336, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.4.attn_q.weight", "offset": 1185792000, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.4.attn_v.weight", "offset": 1207025664, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.5.ffn_gate.weight", "offset": 1210564608, "shape": [ 6144, 16384 ], "size": 3538944, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 1267187712, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.5.attn_k.weight", "offset": 1323810816, "shape": [ 6144, 1024 ], "size": 56623104, "type": 2 }, { "name": "blk.5.attn_output.weight", "offset": 1327349760, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.5.attn_q.weight", "offset": 1348583424, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.5.attn_v.weight", "offset": 1369817088, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 1373356032, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1373380608, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.10.ffn_gate.weight", "offset": 1430003712, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 1486626816, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 1543249920, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1543274496, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.10.attn_output.weight", "offset": 1546813440, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.10.attn_q.weight", "offset": 1568047104, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.10.attn_v.weight", "offset": 1589280768, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 1592819712, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1592844288, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.11.ffn_gate.weight", "offset": 1649467392, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 1706090496, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 1762713600, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1762738176, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.11.attn_output.weight", "offset": 1766277120, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.11.attn_q.weight", "offset": 1787510784, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.11.attn_v.weight", "offset": 1808744448, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.12.attn_k.weight", "offset": 1812283392, "shape": [ 6144, 1024 ], "size": 3538944, "type": 2 }, { "name": "blk.12.attn_q.weight", "offset": 1815822336, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.12.attn_v.weight", "offset": 1837056000, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 1840594944, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1840619520, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 1897242624, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 1897267200, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1897291776, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.6.ffn_gate.weight", "offset": 1953914880, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 2010537984, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 2067161088, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 2067185664, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 2070724608, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.6.attn_q.weight", "offset": 2091958272, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.6.attn_v.weight", "offset": 2113191936, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.7.attn_norm.weight", "offset": 2116730880, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2116755456, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.7.ffn_gate.weight", "offset": 2173378560, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 2230001664, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 2286624768, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2286649344, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.7.attn_output.weight", "offset": 2290188288, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.7.attn_q.weight", "offset": 2311421952, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.7.attn_v.weight", "offset": 2332655616, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 2336194560, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2336219136, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.8.ffn_gate.weight", "offset": 2392842240, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 2449465344, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 2506088448, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2506113024, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.8.attn_output.weight", "offset": 2509651968, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.8.attn_q.weight", "offset": 2530885632, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.8.attn_v.weight", "offset": 2552119296, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 2555658240, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2555682816, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.9.ffn_gate.weight", "offset": 2612305920, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 2668929024, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 2725552128, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2725576704, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.9.attn_output.weight", "offset": 2729115648, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.9.attn_q.weight", "offset": 2750349312, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.9.attn_v.weight", "offset": 2771582976, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 2775121920, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2775146496, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.12.ffn_gate.weight", "offset": 2831769600, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 2888392704, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 2945015808, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2945040384, "shape": [ 6144, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 2966274048, "shape": [ 6144 ], "size": 21233664, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2966298624, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.13.ffn_gate.weight", "offset": 3022921728, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 3079544832, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 3136167936, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 3136192512, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.13.attn_output.weight", "offset": 3139731456, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.13.attn_q.weight", "offset": 3160965120, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.13.attn_v.weight", "offset": 3182198784, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 3185737728, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3185762304, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.14.ffn_gate.weight", "offset": 3242385408, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 3299008512, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 3355631616, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 3355656192, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.14.attn_output.weight", "offset": 3359195136, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.14.attn_q.weight", "offset": 3380428800, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.14.attn_v.weight", "offset": 3401662464, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 3405201408, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3405225984, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.15.ffn_gate.weight", "offset": 3461849088, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 3518472192, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 3575095296, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 3575119872, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.15.attn_output.weight", "offset": 3578658816, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.15.attn_q.weight", "offset": 3599892480, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.15.attn_v.weight", "offset": 3621126144, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 3624665088, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3624689664, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.16.ffn_gate.weight", "offset": 3681312768, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 3737935872, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 3794558976, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 3794583552, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.16.attn_output.weight", "offset": 3798122496, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.16.attn_q.weight", "offset": 3819356160, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.16.attn_v.weight", "offset": 3840589824, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 3844128768, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3844153344, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.17.ffn_gate.weight", "offset": 3900776448, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 3957399552, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 4014022656, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 4014047232, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.17.attn_output.weight", "offset": 4017586176, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.17.attn_q.weight", "offset": 4038819840, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.17.attn_v.weight", "offset": 4060053504, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.18.ffn_gate.weight", "offset": 4063592448, "shape": [ 6144, 16384 ], "size": 3538944, "type": 2 }, { "name": "blk.18.attn_k.weight", "offset": 4120215552, "shape": [ 6144, 1024 ], "size": 56623104, "type": 2 }, { "name": "blk.18.attn_output.weight", "offset": 4123754496, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.18.attn_q.weight", "offset": 4144988160, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.18.attn_v.weight", "offset": 4166221824, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 4169760768, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 4169785344, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 4226408448, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 4283031552, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 4283056128, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4283080704, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.19.ffn_gate.weight", "offset": 4339703808, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 4396326912, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.19.ffn_norm.weight", "offset": 4452950016, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 4452974592, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.19.attn_output.weight", "offset": 4456513536, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.19.attn_q.weight", "offset": 4477747200, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.19.attn_v.weight", "offset": 4498980864, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 4502519808, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4502544384, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.20.ffn_gate.weight", "offset": 4559167488, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 4615790592, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 4672413696, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 4672438272, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.20.attn_output.weight", "offset": 4675977216, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.20.attn_q.weight", "offset": 4697210880, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.20.attn_v.weight", "offset": 4718444544, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 4721983488, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4722008064, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.21.ffn_gate.weight", "offset": 4778631168, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 4835254272, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 4891877376, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 4891901952, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 4895440896, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.21.attn_q.weight", "offset": 4916674560, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.21.attn_v.weight", "offset": 4937908224, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.22.attn_norm.weight", "offset": 4941447168, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4941471744, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.22.ffn_gate.weight", "offset": 4998094848, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.22.ffn_up.weight", "offset": 5054717952, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 5111341056, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 5111365632, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.22.attn_output.weight", "offset": 5114904576, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.22.attn_q.weight", "offset": 5136138240, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.22.attn_v.weight", "offset": 5157371904, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.23.attn_norm.weight", "offset": 5160910848, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5160935424, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.23.ffn_gate.weight", "offset": 5217558528, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 5274181632, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 5330804736, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 5330829312, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.23.attn_output.weight", "offset": 5334368256, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.23.attn_q.weight", "offset": 5355601920, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.23.attn_v.weight", "offset": 5376835584, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.24.ffn_gate.weight", "offset": 5380374528, "shape": [ 6144, 16384 ], "size": 3538944, "type": 2 }, { "name": "blk.24.ffn_up.weight", "offset": 5436997632, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.24.attn_k.weight", "offset": 5493620736, "shape": [ 6144, 1024 ], "size": 56623104, "type": 2 }, { "name": "blk.24.attn_output.weight", "offset": 5497159680, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.24.attn_q.weight", "offset": 5518393344, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.24.attn_v.weight", "offset": 5539627008, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.24.attn_norm.weight", "offset": 5543165952, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5543190528, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.24.ffn_norm.weight", "offset": 5599813632, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.25.attn_norm.weight", "offset": 5599838208, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5599862784, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.25.ffn_gate.weight", "offset": 5656485888, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.25.ffn_up.weight", "offset": 5713108992, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.25.ffn_norm.weight", "offset": 5769732096, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 5769756672, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.25.attn_output.weight", "offset": 5773295616, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.25.attn_q.weight", "offset": 5794529280, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.25.attn_v.weight", "offset": 5815762944, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 5819301888, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5819326464, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.26.ffn_gate.weight", "offset": 5875949568, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.26.ffn_up.weight", "offset": 5932572672, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.26.ffn_norm.weight", "offset": 5989195776, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 5989220352, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.26.attn_output.weight", "offset": 5992759296, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.26.attn_q.weight", "offset": 6013992960, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.26.attn_v.weight", "offset": 6035226624, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 6038765568, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 6038790144, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.27.ffn_gate.weight", "offset": 6095413248, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.27.ffn_up.weight", "offset": 6152036352, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.27.ffn_norm.weight", "offset": 6208659456, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 6208684032, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.27.attn_output.weight", "offset": 6212222976, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.27.attn_q.weight", "offset": 6233456640, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.27.attn_v.weight", "offset": 6254690304, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.28.attn_norm.weight", "offset": 6258229248, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 6258253824, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.28.ffn_gate.weight", "offset": 6314876928, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.28.ffn_up.weight", "offset": 6371500032, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.28.ffn_norm.weight", "offset": 6428123136, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 6428147712, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.28.attn_output.weight", "offset": 6431686656, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.28.attn_q.weight", "offset": 6452920320, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.28.attn_v.weight", "offset": 6474153984, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.29.attn_norm.weight", "offset": 6477692928, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 6477717504, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.29.ffn_gate.weight", "offset": 6534340608, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.29.ffn_up.weight", "offset": 6590963712, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.29.ffn_norm.weight", "offset": 6647586816, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 6647611392, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.29.attn_output.weight", "offset": 6651150336, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.29.attn_q.weight", "offset": 6672384000, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.29.attn_v.weight", "offset": 6693617664, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.30.attn_norm.weight", "offset": 6697156608, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 6697181184, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.30.ffn_gate.weight", "offset": 6753804288, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.30.ffn_up.weight", "offset": 6810427392, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.30.ffn_norm.weight", "offset": 6867050496, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 6867075072, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.30.attn_output.weight", "offset": 6870614016, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.30.attn_q.weight", "offset": 6891847680, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.30.attn_v.weight", "offset": 6913081344, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.31.attn_k.weight", "offset": 6916620288, "shape": [ 6144, 1024 ], "size": 3538944, "type": 2 }, { "name": "blk.31.attn_q.weight", "offset": 6920159232, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.31.attn_v.weight", "offset": 6941392896, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.31.attn_norm.weight", "offset": 6944931840, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 6944956416, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.31.ffn_gate.weight", "offset": 7001579520, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.31.ffn_up.weight", "offset": 7058202624, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.31.ffn_norm.weight", "offset": 7114825728, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 7114850304, "shape": [ 6144, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.32.attn_norm.weight", "offset": 7136083968, "shape": [ 6144 ], "size": 21233664, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 7136108544, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.32.ffn_gate.weight", "offset": 7192731648, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.32.ffn_up.weight", "offset": 7249354752, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.32.ffn_norm.weight", "offset": 7305977856, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 7306002432, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.32.attn_output.weight", "offset": 7309541376, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.32.attn_q.weight", "offset": 7330775040, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.32.attn_v.weight", "offset": 7352008704, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.33.attn_norm.weight", "offset": 7355547648, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 7355572224, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.33.ffn_gate.weight", "offset": 7412195328, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.33.ffn_up.weight", "offset": 7468818432, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.33.ffn_norm.weight", "offset": 7525441536, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 7525466112, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.33.attn_output.weight", "offset": 7529005056, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.33.attn_q.weight", "offset": 7550238720, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.33.attn_v.weight", "offset": 7571472384, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.34.attn_norm.weight", "offset": 7575011328, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 7575035904, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.34.ffn_gate.weight", "offset": 7631659008, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.34.ffn_up.weight", "offset": 7688282112, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.34.ffn_norm.weight", "offset": 7744905216, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 7744929792, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.34.attn_output.weight", "offset": 7748468736, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.34.attn_q.weight", "offset": 7769702400, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.34.attn_v.weight", "offset": 7790936064, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.35.attn_norm.weight", "offset": 7794475008, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 7794499584, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.35.ffn_gate.weight", "offset": 7851122688, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.35.ffn_up.weight", "offset": 7907745792, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.35.ffn_norm.weight", "offset": 7964368896, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 7964393472, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.35.attn_output.weight", "offset": 7967932416, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.35.attn_q.weight", "offset": 7989166080, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.35.attn_v.weight", "offset": 8010399744, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.36.attn_norm.weight", "offset": 8013938688, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 8013963264, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.36.ffn_gate.weight", "offset": 8070586368, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.36.ffn_up.weight", "offset": 8127209472, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.36.ffn_norm.weight", "offset": 8183832576, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 8183857152, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.36.attn_output.weight", "offset": 8187396096, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.36.attn_q.weight", "offset": 8208629760, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.36.attn_v.weight", "offset": 8229863424, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.37.ffn_gate.weight", "offset": 8233402368, "shape": [ 6144, 16384 ], "size": 3538944, "type": 2 }, { "name": "blk.37.attn_k.weight", "offset": 8290025472, "shape": [ 6144, 1024 ], "size": 56623104, "type": 2 }, { "name": "blk.37.attn_output.weight", "offset": 8293564416, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.37.attn_q.weight", "offset": 8314798080, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.37.attn_v.weight", "offset": 8336031744, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.37.attn_norm.weight", "offset": 8339570688, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 8339595264, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.37.ffn_up.weight", "offset": 8396218368, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.37.ffn_norm.weight", "offset": 8452841472, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.38.attn_norm.weight", "offset": 8452866048, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 8452890624, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.38.ffn_gate.weight", "offset": 8509513728, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.38.ffn_up.weight", "offset": 8566136832, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.38.ffn_norm.weight", "offset": 8622759936, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 8622784512, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.38.attn_output.weight", "offset": 8626323456, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.38.attn_q.weight", "offset": 8647557120, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.38.attn_v.weight", "offset": 8668790784, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.39.attn_norm.weight", "offset": 8672329728, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 8672354304, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.39.ffn_gate.weight", "offset": 8728977408, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.39.ffn_up.weight", "offset": 8785600512, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.39.ffn_norm.weight", "offset": 8842223616, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 8842248192, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.39.attn_output.weight", "offset": 8845787136, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.39.attn_q.weight", "offset": 8867020800, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.39.attn_v.weight", "offset": 8888254464, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.40.attn_norm.weight", "offset": 8891793408, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 8891817984, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.40.ffn_gate.weight", "offset": 8948441088, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.40.ffn_up.weight", "offset": 9005064192, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.40.ffn_norm.weight", "offset": 9061687296, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 9061711872, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.40.attn_output.weight", "offset": 9065250816, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.40.attn_q.weight", "offset": 9086484480, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.40.attn_v.weight", "offset": 9107718144, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.41.attn_norm.weight", "offset": 9111257088, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 9111281664, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.41.ffn_gate.weight", "offset": 9167904768, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.41.ffn_up.weight", "offset": 9224527872, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.41.ffn_norm.weight", "offset": 9281150976, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 9281175552, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.41.attn_output.weight", "offset": 9284714496, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.41.attn_q.weight", "offset": 9305948160, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.41.attn_v.weight", "offset": 9327181824, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.42.attn_norm.weight", "offset": 9330720768, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 9330745344, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.42.ffn_gate.weight", "offset": 9387368448, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.42.ffn_up.weight", "offset": 9443991552, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.42.ffn_norm.weight", "offset": 9500614656, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 9500639232, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.42.attn_output.weight", "offset": 9504178176, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.42.attn_q.weight", "offset": 9525411840, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.42.attn_v.weight", "offset": 9546645504, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.43.ffn_gate.weight", "offset": 9550184448, "shape": [ 6144, 16384 ], "size": 3538944, "type": 2 }, { "name": "blk.43.ffn_up.weight", "offset": 9606807552, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.43.attn_k.weight", "offset": 9663430656, "shape": [ 6144, 1024 ], "size": 56623104, "type": 2 }, { "name": "blk.43.attn_output.weight", "offset": 9666969600, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.43.attn_q.weight", "offset": 9688203264, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.43.attn_v.weight", "offset": 9709436928, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.43.attn_norm.weight", "offset": 9712975872, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 9713000448, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.43.ffn_norm.weight", "offset": 9769623552, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 9769648128, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 9769672704, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.44.ffn_gate.weight", "offset": 9826295808, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.44.ffn_up.weight", "offset": 9882918912, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.44.ffn_norm.weight", "offset": 9939542016, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 9939566592, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.44.attn_output.weight", "offset": 9943105536, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.44.attn_q.weight", "offset": 9964339200, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.44.attn_v.weight", "offset": 9985572864, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.45.attn_norm.weight", "offset": 9989111808, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 9989136384, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.45.ffn_gate.weight", "offset": 10045759488, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.45.ffn_up.weight", "offset": 10102382592, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.45.ffn_norm.weight", "offset": 10159005696, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 10159030272, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.45.attn_output.weight", "offset": 10162569216, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.45.attn_q.weight", "offset": 10183802880, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.45.attn_v.weight", "offset": 10205036544, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.46.attn_norm.weight", "offset": 10208575488, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 10208600064, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.46.ffn_gate.weight", "offset": 10265223168, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.46.ffn_up.weight", "offset": 10321846272, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.46.ffn_norm.weight", "offset": 10378469376, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 10378493952, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.46.attn_output.weight", "offset": 10382032896, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.46.attn_q.weight", "offset": 10403266560, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.46.attn_v.weight", "offset": 10424500224, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.47.attn_norm.weight", "offset": 10428039168, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 10428063744, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.47.ffn_gate.weight", "offset": 10484686848, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.47.ffn_up.weight", "offset": 10541309952, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.47.ffn_norm.weight", "offset": 10597933056, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 10597957632, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.47.attn_output.weight", "offset": 10601496576, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.47.attn_q.weight", "offset": 10622730240, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.47.attn_v.weight", "offset": 10643963904, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.48.attn_norm.weight", "offset": 10647502848, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 10647527424, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.48.ffn_gate.weight", "offset": 10704150528, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.48.ffn_up.weight", "offset": 10760773632, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.48.ffn_norm.weight", "offset": 10817396736, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 10817421312, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.48.attn_output.weight", "offset": 10820960256, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.48.attn_q.weight", "offset": 10842193920, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.48.attn_v.weight", "offset": 10863427584, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.49.attn_norm.weight", "offset": 10866966528, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 10866991104, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.49.ffn_gate.weight", "offset": 10923614208, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.49.ffn_up.weight", "offset": 10980237312, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.49.ffn_norm.weight", "offset": 11036860416, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 11036884992, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.49.attn_output.weight", "offset": 11040423936, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.49.attn_q.weight", "offset": 11061657600, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.49.attn_v.weight", "offset": 11082891264, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.50.attn_k.weight", "offset": 11086430208, "shape": [ 6144, 1024 ], "size": 3538944, "type": 2 }, { "name": "blk.50.attn_q.weight", "offset": 11089969152, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.50.attn_v.weight", "offset": 11111202816, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "output.weight", "offset": 11114741760, "shape": [ 6144, 32768 ], "size": 3538944, "type": 14 }, { "name": "blk.50.attn_norm.weight", "offset": 11279892480, "shape": [ 6144 ], "size": 165150720, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 11279917056, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.50.ffn_gate.weight", "offset": 11336540160, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.50.ffn_up.weight", "offset": 11393163264, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.50.ffn_norm.weight", "offset": 11449786368, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 11449810944, "shape": [ 6144, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.51.attn_norm.weight", "offset": 11471044608, "shape": [ 6144 ], "size": 21233664, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 11471069184, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.51.ffn_gate.weight", "offset": 11527692288, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.51.ffn_up.weight", "offset": 11584315392, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.51.ffn_norm.weight", "offset": 11640938496, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 11640963072, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.51.attn_output.weight", "offset": 11644502016, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.51.attn_q.weight", "offset": 11665735680, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.51.attn_v.weight", "offset": 11686969344, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.52.attn_norm.weight", "offset": 11690508288, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 11690532864, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.52.ffn_gate.weight", "offset": 11747155968, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.52.ffn_up.weight", "offset": 11803779072, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.52.ffn_norm.weight", "offset": 11860402176, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 11860426752, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.52.attn_output.weight", "offset": 11863965696, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.52.attn_q.weight", "offset": 11885199360, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.52.attn_v.weight", "offset": 11906433024, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.53.attn_norm.weight", "offset": 11909971968, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 11909996544, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.53.ffn_gate.weight", "offset": 11966619648, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.53.ffn_up.weight", "offset": 12023242752, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.53.ffn_norm.weight", "offset": 12079865856, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 12079890432, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.53.attn_output.weight", "offset": 12083429376, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.53.attn_q.weight", "offset": 12104663040, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.53.attn_v.weight", "offset": 12125896704, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.54.attn_norm.weight", "offset": 12129435648, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 12129460224, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.54.ffn_gate.weight", "offset": 12186083328, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.54.ffn_up.weight", "offset": 12242706432, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.54.ffn_norm.weight", "offset": 12299329536, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 12299354112, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.54.attn_output.weight", "offset": 12302893056, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.54.attn_q.weight", "offset": 12324126720, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.54.attn_v.weight", "offset": 12345360384, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "blk.55.attn_norm.weight", "offset": 12348899328, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 12348923904, "shape": [ 16384, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.55.ffn_gate.weight", "offset": 12405547008, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.55.ffn_up.weight", "offset": 12462170112, "shape": [ 6144, 16384 ], "size": 56623104, "type": 2 }, { "name": "blk.55.ffn_norm.weight", "offset": 12518793216, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 12518817792, "shape": [ 6144, 1024 ], "size": 24576, "type": 2 }, { "name": "blk.55.attn_output.weight", "offset": 12522356736, "shape": [ 6144, 6144 ], "size": 3538944, "type": 2 }, { "name": "blk.55.attn_q.weight", "offset": 12543590400, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.55.attn_v.weight", "offset": 12564824064, "shape": [ 6144, 1024 ], "size": 21233664, "type": 2 }, { "name": "output_norm.weight", "offset": 12568363008, "shape": [ 6144 ], "size": 3538944, "type": 0 } ], "version": 3 }