codestral

{ "metadata": { "general.architecture": "llama", "general.file_type": 11, "general.name": "Codestral-22B-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 32768, "llama.embedding_length": 6144, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_space_prefix": true, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 22247282688, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 86507520, "shape": [ 6144 ], "size": 86507520, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 86532096, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 129785856, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 173039616, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 216293376, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 216317952, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 219021312, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.0.attn_q.weight", "offset": 235241472, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 251461632, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 254164992, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 254189568, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.1.ffn_gate.weight", "offset": 297443328, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 340697088, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 383950848, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 383975424, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 386678784, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.1.attn_q.weight", "offset": 402898944, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 419119104, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 421822464, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 421847040, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.2.ffn_gate.weight", "offset": 465100800, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 508354560, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 551608320, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 551632896, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 554336256, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.2.attn_q.weight", "offset": 570556416, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.2.attn_v.weight", "offset": 586776576, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 589479936, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 589504512, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.3.ffn_gate.weight", "offset": 632758272, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 676012032, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 719265792, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 719290368, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 721993728, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.3.attn_q.weight", "offset": 738213888, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.3.attn_v.weight", "offset": 754434048, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 757137408, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 757161984, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.4.ffn_gate.weight", "offset": 800415744, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 843669504, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 886923264, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 886947840, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 889651200, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.4.attn_q.weight", "offset": 905871360, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 922091520, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.5.ffn_gate.weight", "offset": 924794880, "shape": [ 6144, 16384 ], "size": 2703360, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 968048640, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.5.attn_k.weight", "offset": 1011302400, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 1014005760, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.5.attn_q.weight", "offset": 1030225920, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 1046446080, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.10.attn_norm.weight", "offset": 1049149440, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1049174016, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.10.ffn_gate.weight", "offset": 1092427776, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 1135681536, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 1178935296, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1178959872, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 1181663232, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.10.attn_q.weight", "offset": 1197883392, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 1214103552, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 1216806912, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1216831488, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.11.ffn_gate.weight", "offset": 1260085248, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1303339008, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 1346592768, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1346617344, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 1349320704, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.11.attn_q.weight", "offset": 1365540864, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.11.attn_v.weight", "offset": 1381761024, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.12.attn_k.weight", "offset": 1384464384, "shape": [ 6144, 1024 ], "size": 2703360, "type": 11 }, { "name": "blk.12.attn_q.weight", "offset": 1387167744, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.12.attn_v.weight", "offset": 1403387904, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 1406091264, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1406115840, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 1449369600, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 1449394176, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1449418752, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.6.ffn_gate.weight", "offset": 1492672512, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 1535926272, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 1579180032, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1579204608, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 1581907968, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.6.attn_q.weight", "offset": 1598128128, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 1614348288, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.7.attn_norm.weight", "offset": 1617051648, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1617076224, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.7.ffn_gate.weight", "offset": 1660329984, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 1703583744, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 1746837504, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1746862080, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 1749565440, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.7.attn_q.weight", "offset": 1765785600, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 1782005760, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 1784709120, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1784733696, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.8.ffn_gate.weight", "offset": 1827987456, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 1871241216, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 1914494976, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1914519552, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 1917222912, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.8.attn_q.weight", "offset": 1933443072, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 1949663232, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 1952366592, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1952391168, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.9.ffn_gate.weight", "offset": 1995644928, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 2038898688, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 2082152448, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2082177024, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 2084880384, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.9.attn_q.weight", "offset": 2101100544, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.9.attn_v.weight", "offset": 2117320704, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 2120024064, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2120048640, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 2163302400, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 2206556160, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 2249809920, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2249834496, "shape": [ 6144, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 2266054656, "shape": [ 6144 ], "size": 16220160, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2266079232, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 2309332992, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 2352586752, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 2395840512, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2395865088, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 2398568448, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.13.attn_q.weight", "offset": 2414788608, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.13.attn_v.weight", "offset": 2431008768, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 2433712128, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2433736704, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.14.ffn_gate.weight", "offset": 2476990464, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 2520244224, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 2563497984, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2563522560, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 2566225920, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.14.attn_q.weight", "offset": 2582446080, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 2598666240, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 2601369600, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2601394176, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.15.ffn_gate.weight", "offset": 2644647936, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 2687901696, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 2731155456, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2731180032, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 2733883392, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.15.attn_q.weight", "offset": 2750103552, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 2766323712, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 2769027072, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2769051648, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 2812305408, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 2855559168, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 2898812928, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2898837504, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 2901540864, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.16.attn_q.weight", "offset": 2917761024, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 2933981184, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 2936684544, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2936709120, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 2979962880, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 3023216640, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 3066470400, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 3066494976, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 3069198336, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.17.attn_q.weight", "offset": 3085418496, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.17.attn_v.weight", "offset": 3101638656, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.18.ffn_gate.weight", "offset": 3104342016, "shape": [ 6144, 16384 ], "size": 2703360, "type": 11 }, { "name": "blk.18.attn_k.weight", "offset": 3147595776, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 3150299136, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.18.attn_q.weight", "offset": 3166519296, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 3182739456, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 3185442816, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3185467392, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 3228721152, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 3271974912, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 3271999488, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3272024064, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.19.ffn_gate.weight", "offset": 3315277824, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 3358531584, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 3401785344, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 3401809920, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 3404513280, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.19.attn_q.weight", "offset": 3420733440, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 3436953600, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 3439656960, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3439681536, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.20.ffn_gate.weight", "offset": 3482935296, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 3526189056, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 3569442816, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 3569467392, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 3572170752, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.20.attn_q.weight", "offset": 3588390912, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 3604611072, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 3607314432, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3607339008, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.21.ffn_gate.weight", "offset": 3650592768, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 3693846528, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 3737100288, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 3737124864, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 3739828224, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.21.attn_q.weight", "offset": 3756048384, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 3772268544, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 3774971904, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3774996480, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.22.ffn_gate.weight", "offset": 3818250240, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 3861504000, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 3904757760, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3904782336, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 3907485696, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.22.attn_q.weight", "offset": 3923705856, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 3939926016, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.23.attn_norm.weight", "offset": 3942629376, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3942653952, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.23.ffn_gate.weight", "offset": 3985907712, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 4029161472, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 4072415232, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 4072439808, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 4075143168, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.23.attn_q.weight", "offset": 4091363328, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 4107583488, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.24.ffn_gate.weight", "offset": 4110286848, "shape": [ 6144, 16384 ], "size": 2703360, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 4153540608, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.24.attn_k.weight", "offset": 4196794368, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 4199497728, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.24.attn_q.weight", "offset": 4215717888, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 4231938048, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.24.attn_norm.weight", "offset": 4234641408, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4234665984, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 4277919744, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.25.attn_norm.weight", "offset": 4277944320, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4277968896, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.25.ffn_gate.weight", "offset": 4321222656, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 4364476416, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 4407730176, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 4407754752, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 4410458112, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.25.attn_q.weight", "offset": 4426678272, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 4442898432, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 4445601792, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4445626368, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.26.ffn_gate.weight", "offset": 4488880128, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 4532133888, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 4575387648, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 4575412224, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 4578115584, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.26.attn_q.weight", "offset": 4594335744, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 4610555904, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 4613259264, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4613283840, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.27.ffn_gate.weight", "offset": 4656537600, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 4699791360, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 4743045120, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 4743069696, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 4745773056, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.27.attn_q.weight", "offset": 4761993216, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 4778213376, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 4780916736, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 4780941312, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.28.ffn_gate.weight", "offset": 4824195072, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 4867448832, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 4910702592, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 4910727168, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 4913430528, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.28.attn_q.weight", "offset": 4929650688, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 4945870848, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.29.attn_norm.weight", "offset": 4948574208, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 4948598784, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.29.ffn_gate.weight", "offset": 4991852544, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 5035106304, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 5078360064, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 5078384640, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 5081088000, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.29.attn_q.weight", "offset": 5097308160, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 5113528320, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.30.attn_norm.weight", "offset": 5116231680, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 5116256256, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.30.ffn_gate.weight", "offset": 5159510016, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 5202763776, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 5246017536, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 5246042112, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 5248745472, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.30.attn_q.weight", "offset": 5264965632, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 5281185792, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.31.attn_k.weight", "offset": 5283889152, "shape": [ 6144, 1024 ], "size": 2703360, "type": 11 }, { "name": "blk.31.attn_q.weight", "offset": 5286592512, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 5302812672, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.31.attn_norm.weight", "offset": 5305516032, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 5305540608, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.31.ffn_gate.weight", "offset": 5348794368, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 5392048128, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 5435301888, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 5435326464, "shape": [ 6144, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.32.attn_norm.weight", "offset": 5451546624, "shape": [ 6144 ], "size": 16220160, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 5451571200, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.32.ffn_gate.weight", "offset": 5494824960, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 5538078720, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 5581332480, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 5581357056, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 5584060416, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.32.attn_q.weight", "offset": 5600280576, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.32.attn_v.weight", "offset": 5616500736, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.33.attn_norm.weight", "offset": 5619204096, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 5619228672, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.33.ffn_gate.weight", "offset": 5662482432, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 5705736192, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 5748989952, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 5749014528, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 5751717888, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.33.attn_q.weight", "offset": 5767938048, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.33.attn_v.weight", "offset": 5784158208, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.34.attn_norm.weight", "offset": 5786861568, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 5786886144, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.34.ffn_gate.weight", "offset": 5830139904, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 5873393664, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 5916647424, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 5916672000, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 5919375360, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.34.attn_q.weight", "offset": 5935595520, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.34.attn_v.weight", "offset": 5951815680, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.35.attn_norm.weight", "offset": 5954519040, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 5954543616, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.35.ffn_gate.weight", "offset": 5997797376, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 6041051136, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 6084304896, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 6084329472, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 6087032832, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.35.attn_q.weight", "offset": 6103252992, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.35.attn_v.weight", "offset": 6119473152, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.36.attn_norm.weight", "offset": 6122176512, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 6122201088, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.36.ffn_gate.weight", "offset": 6165454848, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 6208708608, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 6251962368, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 6251986944, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 6254690304, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.36.attn_q.weight", "offset": 6270910464, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.36.attn_v.weight", "offset": 6287130624, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.37.ffn_gate.weight", "offset": 6289833984, "shape": [ 6144, 16384 ], "size": 2703360, "type": 11 }, { "name": "blk.37.attn_k.weight", "offset": 6333087744, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 6335791104, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.37.attn_q.weight", "offset": 6352011264, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.37.attn_v.weight", "offset": 6368231424, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.37.attn_norm.weight", "offset": 6370934784, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 6370959360, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 6414213120, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 6457466880, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.38.attn_norm.weight", "offset": 6457491456, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 6457516032, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.38.ffn_gate.weight", "offset": 6500769792, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 6544023552, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 6587277312, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 6587301888, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 6590005248, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.38.attn_q.weight", "offset": 6606225408, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.38.attn_v.weight", "offset": 6622445568, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.39.attn_norm.weight", "offset": 6625148928, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 6625173504, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.39.ffn_gate.weight", "offset": 6668427264, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 6711681024, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 6754934784, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 6754959360, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 6757662720, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.39.attn_q.weight", "offset": 6773882880, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.39.attn_v.weight", "offset": 6790103040, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.40.attn_norm.weight", "offset": 6792806400, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 6792830976, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.40.ffn_gate.weight", "offset": 6836084736, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.40.ffn_up.weight", "offset": 6879338496, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.40.ffn_norm.weight", "offset": 6922592256, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 6922616832, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 6925320192, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.40.attn_q.weight", "offset": 6941540352, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.40.attn_v.weight", "offset": 6957760512, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.41.attn_norm.weight", "offset": 6960463872, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 6960488448, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.41.ffn_gate.weight", "offset": 7003742208, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.41.ffn_up.weight", "offset": 7046995968, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.41.ffn_norm.weight", "offset": 7090249728, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 7090274304, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 7092977664, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.41.attn_q.weight", "offset": 7109197824, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.41.attn_v.weight", "offset": 7125417984, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.42.attn_norm.weight", "offset": 7128121344, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 7128145920, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.42.ffn_gate.weight", "offset": 7171399680, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.42.ffn_up.weight", "offset": 7214653440, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.42.ffn_norm.weight", "offset": 7257907200, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 7257931776, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.42.attn_output.weight", "offset": 7260635136, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.42.attn_q.weight", "offset": 7276855296, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.42.attn_v.weight", "offset": 7293075456, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.43.ffn_gate.weight", "offset": 7295778816, "shape": [ 6144, 16384 ], "size": 2703360, "type": 11 }, { "name": "blk.43.ffn_up.weight", "offset": 7339032576, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.43.attn_k.weight", "offset": 7382286336, "shape": [ 6144, 1024 ], "size": 43253760, "type": 11 }, { "name": "blk.43.attn_output.weight", "offset": 7384989696, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.43.attn_q.weight", "offset": 7401209856, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.43.attn_v.weight", "offset": 7417430016, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.43.attn_norm.weight", "offset": 7420133376, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 7420157952, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.43.ffn_norm.weight", "offset": 7463411712, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 7463436288, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 7463460864, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.44.ffn_gate.weight", "offset": 7506714624, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.44.ffn_up.weight", "offset": 7549968384, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.44.ffn_norm.weight", "offset": 7593222144, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 7593246720, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.44.attn_output.weight", "offset": 7595950080, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.44.attn_q.weight", "offset": 7612170240, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.44.attn_v.weight", "offset": 7628390400, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.45.attn_norm.weight", "offset": 7631093760, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 7631118336, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.45.ffn_gate.weight", "offset": 7674372096, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.45.ffn_up.weight", "offset": 7717625856, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.45.ffn_norm.weight", "offset": 7760879616, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 7760904192, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.45.attn_output.weight", "offset": 7763607552, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.45.attn_q.weight", "offset": 7779827712, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.45.attn_v.weight", "offset": 7796047872, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.46.attn_norm.weight", "offset": 7798751232, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 7798775808, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.46.ffn_gate.weight", "offset": 7842029568, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.46.ffn_up.weight", "offset": 7885283328, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.46.ffn_norm.weight", "offset": 7928537088, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 7928561664, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.46.attn_output.weight", "offset": 7931265024, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.46.attn_q.weight", "offset": 7947485184, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.46.attn_v.weight", "offset": 7963705344, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.47.attn_norm.weight", "offset": 7966408704, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 7966433280, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.47.ffn_gate.weight", "offset": 8009687040, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.47.ffn_up.weight", "offset": 8052940800, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.47.ffn_norm.weight", "offset": 8096194560, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 8096219136, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.47.attn_output.weight", "offset": 8098922496, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.47.attn_q.weight", "offset": 8115142656, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.47.attn_v.weight", "offset": 8131362816, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.48.attn_norm.weight", "offset": 8134066176, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 8134090752, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.48.ffn_gate.weight", "offset": 8177344512, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.48.ffn_up.weight", "offset": 8220598272, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.48.ffn_norm.weight", "offset": 8263852032, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 8263876608, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.48.attn_output.weight", "offset": 8266579968, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.48.attn_q.weight", "offset": 8282800128, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.48.attn_v.weight", "offset": 8299020288, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.49.attn_norm.weight", "offset": 8301723648, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 8301748224, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.49.ffn_gate.weight", "offset": 8345001984, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.49.ffn_up.weight", "offset": 8388255744, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.49.ffn_norm.weight", "offset": 8431509504, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 8431534080, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.49.attn_output.weight", "offset": 8434237440, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.49.attn_q.weight", "offset": 8450457600, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.49.attn_v.weight", "offset": 8466677760, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.50.attn_k.weight", "offset": 8469381120, "shape": [ 6144, 1024 ], "size": 2703360, "type": 11 }, { "name": "blk.50.attn_q.weight", "offset": 8472084480, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.50.attn_v.weight", "offset": 8488304640, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "output.weight", "offset": 8491008000, "shape": [ 6144, 32768 ], "size": 2703360, "type": 14 }, { "name": "blk.50.attn_norm.weight", "offset": 8656158720, "shape": [ 6144 ], "size": 165150720, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 8656183296, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.50.ffn_gate.weight", "offset": 8699437056, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.50.ffn_up.weight", "offset": 8742690816, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.50.ffn_norm.weight", "offset": 8785944576, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 8785969152, "shape": [ 6144, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.51.attn_norm.weight", "offset": 8802189312, "shape": [ 6144 ], "size": 16220160, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 8802213888, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.51.ffn_gate.weight", "offset": 8845467648, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.51.ffn_up.weight", "offset": 8888721408, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.51.ffn_norm.weight", "offset": 8931975168, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 8931999744, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.51.attn_output.weight", "offset": 8934703104, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.51.attn_q.weight", "offset": 8950923264, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.51.attn_v.weight", "offset": 8967143424, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.52.attn_norm.weight", "offset": 8969846784, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 8969871360, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.52.ffn_gate.weight", "offset": 9013125120, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.52.ffn_up.weight", "offset": 9056378880, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.52.ffn_norm.weight", "offset": 9099632640, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 9099657216, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.52.attn_output.weight", "offset": 9102360576, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.52.attn_q.weight", "offset": 9118580736, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.52.attn_v.weight", "offset": 9134800896, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.53.attn_norm.weight", "offset": 9137504256, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 9137528832, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.53.ffn_gate.weight", "offset": 9180782592, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.53.ffn_up.weight", "offset": 9224036352, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.53.ffn_norm.weight", "offset": 9267290112, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 9267314688, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.53.attn_output.weight", "offset": 9270018048, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.53.attn_q.weight", "offset": 9286238208, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.53.attn_v.weight", "offset": 9302458368, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.54.attn_norm.weight", "offset": 9305161728, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 9305186304, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.54.ffn_gate.weight", "offset": 9348440064, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.54.ffn_up.weight", "offset": 9391693824, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.54.ffn_norm.weight", "offset": 9434947584, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 9434972160, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.54.attn_output.weight", "offset": 9437675520, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.54.attn_q.weight", "offset": 9453895680, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.54.attn_v.weight", "offset": 9470115840, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "blk.55.attn_norm.weight", "offset": 9472819200, "shape": [ 6144 ], "size": 2703360, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 9472843776, "shape": [ 16384, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.55.ffn_gate.weight", "offset": 9516097536, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.55.ffn_up.weight", "offset": 9559351296, "shape": [ 6144, 16384 ], "size": 43253760, "type": 11 }, { "name": "blk.55.ffn_norm.weight", "offset": 9602605056, "shape": [ 6144 ], "size": 43253760, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 9602629632, "shape": [ 6144, 1024 ], "size": 24576, "type": 11 }, { "name": "blk.55.attn_output.weight", "offset": 9605332992, "shape": [ 6144, 6144 ], "size": 2703360, "type": 11 }, { "name": "blk.55.attn_q.weight", "offset": 9621553152, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.55.attn_v.weight", "offset": 9637773312, "shape": [ 6144, 1024 ], "size": 16220160, "type": 11 }, { "name": "output_norm.weight", "offset": 9640476672, "shape": [ 6144 ], "size": 2703360, "type": 0 } ], "version": 3 }