codestral

{ "metadata": { "general.architecture": "llama", "general.file_type": 15, "general.name": "Codestral-22B-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 32768, "llama.embedding_length": 6144, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_space_prefix": true, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 22247282688, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 113246208, "shape": [ 6144 ], "size": 113246208, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 113270784, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 195846144, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 252469248, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.0.ffn_norm.weight", "offset": 309092352, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 309116928, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 312655872, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 333889536, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.0.attn_v.weight", "offset": 355123200, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 360284160, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 360308736, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 442884096, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 499507200, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 556130304, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 556154880, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 559693824, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 580927488, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.1.attn_v.weight", "offset": 602161152, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 607322112, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 607346688, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 689922048, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 746545152, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 803168256, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 803192832, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 806731776, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 827965440, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.2.attn_v.weight", "offset": 849199104, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 854360064, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 854384640, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 936960000, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 993583104, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 1050206208, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1050230784, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 1053769728, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 1075003392, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.3.attn_v.weight", "offset": 1096237056, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1101398016, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1101422592, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 1183997952, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 1240621056, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 1297244160, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1297268736, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 1300807680, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 1322041344, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.4.attn_v.weight", "offset": 1343275008, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.5.ffn_gate.weight", "offset": 1348435968, "shape": [ 6144, 16384 ], "size": 5160960, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 1405059072, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.5.attn_k.weight", "offset": 1461682176, "shape": [ 6144, 1024 ], "size": 56623104, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 1465221120, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 1486454784, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.5.attn_v.weight", "offset": 1507688448, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 1512849408, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1512873984, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 1595449344, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 1652072448, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 1708695552, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1708720128, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 1712259072, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.10.attn_q.weight", "offset": 1733492736, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.10.attn_v.weight", "offset": 1754726400, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 1759887360, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1759911936, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.11.ffn_gate.weight", "offset": 1842487296, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 1899110400, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 1955733504, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1955758080, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 1959297024, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.11.attn_q.weight", "offset": 1980530688, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.11.attn_v.weight", "offset": 2001764352, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.12.attn_k.weight", "offset": 2005303296, "shape": [ 6144, 1024 ], "size": 3538944, "type": 12 }, { "name": "blk.12.attn_q.weight", "offset": 2008842240, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.12.attn_v.weight", "offset": 2030075904, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 2033614848, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 2033639424, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 2090262528, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 2090287104, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2090311680, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 2146934784, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 2203557888, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 2260180992, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 2260205568, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 2263744512, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 2284978176, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.6.attn_v.weight", "offset": 2306211840, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 2311372800, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2311397376, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 2393972736, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 2450595840, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 2507218944, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2507243520, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 2510782464, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 2532016128, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "\u0001\u0000\u0000\u0000\u0000\u0018\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffdD", "offset": 2553249792, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 2556788736, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2556813312, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 2613436416, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 2670059520, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 2726682624, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2726707200, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 2730246144, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 2751479808, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.8.attn_v.weight", "offset": 2772713472, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 2776252416, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2776276992, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 2832900096, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 2889523200, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.9.ffn_norm.weight", "offset": 2946146304, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2946170880, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 2949709824, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 2970943488, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.9.attn_v.weight", "offset": 2992177152, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 2997338112, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2997362688, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.12.ffn_gate.weight", "offset": 3079938048, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 3136561152, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 3193184256, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 3193208832, "shape": [ 6144, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 3214442496, "shape": [ 6144 ], "size": 21233664, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 3214467072, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 3271090176, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 3327713280, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 3384336384, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 3384360960, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 3387899904, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.13.attn_q.weight", "offset": 3409133568, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.13.attn_v.weight", "offset": 3430367232, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 3433906176, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3433930752, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 3490553856, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 3547176960, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 3603800064, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 3603824640, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 3607363584, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 3628597248, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.14.attn_v.weight", "offset": 3649830912, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 3653369856, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3653394432, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.15.ffn_gate.weight", "offset": 3735969792, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 3792592896, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 3849216000, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 3849240576, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 3852779520, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 3874013184, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.15.attn_v.weight", "offset": 3895246848, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 3900407808, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3900432384, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 3957055488, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 4013678592, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 4070301696, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 4070326272, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 4073865216, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.16.attn_q.weight", "offset": 4095098880, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.16.attn_v.weight", "offset": 4116332544, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 4119871488, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4119896064, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 4176519168, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 4233142272, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 4289765376, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 4289789952, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 4293328896, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 4314562560, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.17.attn_v.weight", "offset": 4335796224, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 4339335168, "shape": [ 6144, 16384 ], "size": 3538944, "type": 12 }, { "name": "blk.18.attn_k.weight", "offset": 4395958272, "shape": [ 6144, 1024 ], "size": 56623104, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 4399497216, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.18.attn_q.weight", "offset": 4420730880, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.18.attn_v.weight", "offset": 4441964544, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 4447125504, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 4447150080, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.18.ffn_up.weight", "offset": 4529725440, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 4586348544, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 4586373120, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4586397696, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 4643020800, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 4699643904, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.19.ffn_norm.weight", "offset": 4756267008, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 4756291584, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 4759830528, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 4781064192, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.19.attn_v.weight", "offset": 4802297856, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 4805836800, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4805861376, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 4862484480, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 4919107584, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 4975730688, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 4975755264, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 4979294208, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 5000527872, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.20.attn_v.weight", "offset": 5021761536, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 5025300480, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5025325056, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.21.ffn_gate.weight", "offset": 5107900416, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 5164523520, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 5221146624, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 5221171200, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 5224710144, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 5245943808, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.21.attn_v.weight", "offset": 5267177472, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 5272338432, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5272363008, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 5328986112, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 5385609216, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 5442232320, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 5442256896, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 5445795840, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 5467029504, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.22.attn_v.weight", "offset": 5488263168, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 5491802112, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5491826688, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 5548449792, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 5605072896, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 5661696000, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 5661720576, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 5665259520, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 5686493184, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.23.attn_v.weight", "offset": 5707726848, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 5711265792, "shape": [ 6144, 16384 ], "size": 3538944, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 5767888896, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.24.attn_k.weight", "offset": 5824512000, "shape": [ 6144, 1024 ], "size": 56623104, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 5828050944, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 5849284608, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.24.attn_v.weight", "offset": 5870518272, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 5875679232, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5875703808, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.24.ffn_norm.weight", "offset": 5958279168, "shape": [ 6144 ], "size": 82575360, "type": 0 }, { "name": "blk.25.attn_norm.weight", "offset": 5958303744, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5958328320, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 6014951424, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 6071574528, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 6128197632, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 6128222208, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 6131761152, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 6152994816, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.25.attn_v.weight", "offset": 6174228480, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 6177767424, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 6177792000, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 6234415104, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 6291038208, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 6347661312, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 6347685888, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 6351224832, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 6372458496, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.26.attn_v.weight", "offset": 6393692160, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 6397231104, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 6397255680, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 6479831040, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 6536454144, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.27.ffn_norm.weight", "offset": 6593077248, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 6593101824, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 6596640768, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.27.attn_q.weight", "offset": 6617874432, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.27.attn_v.weight", "offset": 6639108096, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.28.attn_norm.weight", "offset": 6644269056, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 6644293632, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.28.ffn_gate.weight", "offset": 6700916736, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 6757539840, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.28.ffn_norm.weight", "offset": 6814162944, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 6814187520, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 6817726464, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.28.attn_q.weight", "offset": 6838960128, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.28.attn_v.weight", "offset": 6860193792, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 6863732736, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 6863757312, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.29.ffn_gate.weight", "offset": 6920380416, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 6977003520, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.29.ffn_norm.weight", "offset": 7033626624, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 7033651200, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 7037190144, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.29.attn_q.weight", "offset": 7058423808, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.29.attn_v.weight", "offset": 7079657472, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 7083196416, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 7083220992, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.30.ffn_gate.weight", "offset": 7165796352, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 7222419456, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.30.ffn_norm.weight", "offset": 7279042560, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 7279067136, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 7282606080, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.30.attn_q.weight", "offset": 7303839744, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.30.attn_v.weight", "offset": 7325073408, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.31.attn_k.weight", "offset": 7330234368, "shape": [ 6144, 1024 ], "size": 5160960, "type": 12 }, { "name": "blk.31.attn_q.weight", "offset": 7333773312, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.31.attn_v.weight", "offset": 7355006976, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 7358545920, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 7358570496, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.31.ffn_gate.weight", "offset": 7415193600, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 7471816704, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.31.ffn_norm.weight", "offset": 7528439808, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 7528464384, "shape": [ 6144, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 7549698048, "shape": [ 6144 ], "size": 21233664, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 7549722624, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.32.ffn_gate.weight", "offset": 7606345728, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 7662968832, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.32.ffn_norm.weight", "offset": 7719591936, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 7719616512, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.32.attn_output.weight", "offset": 7723155456, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.32.attn_q.weight", "offset": 7744389120, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.32.attn_v.weight", "offset": 7765622784, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 7769161728, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 7769186304, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.33.ffn_gate.weight", "offset": 7851761664, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 7908384768, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.33.ffn_norm.weight", "offset": 7965007872, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 7965032448, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.33.attn_output.weight", "offset": 7968571392, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.33.attn_q.weight", "offset": 7989805056, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.33.attn_v.weight", "offset": 8011038720, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.34.attn_norm.weight", "offset": 8016199680, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 8016224256, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.34.ffn_gate.weight", "offset": 8072847360, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.34.ffn_up.weight", "offset": 8129470464, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.34.ffn_norm.weight", "offset": 8186093568, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 8186118144, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.34.attn_output.weight", "offset": 8189657088, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.34.attn_q.weight", "offset": 8210890752, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.34.attn_v.weight", "offset": 8232124416, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.35.attn_norm.weight", "offset": 8235663360, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 8235687936, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.35.ffn_gate.weight", "offset": 8292311040, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 8348934144, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.35.ffn_norm.weight", "offset": 8405557248, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 8405581824, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 8409120768, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.35.attn_q.weight", "offset": 8430354432, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.35.attn_v.weight", "offset": 8451588096, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.36.attn_norm.weight", "offset": 8455127040, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 8455151616, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.36.ffn_gate.weight", "offset": 8537726976, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 8594350080, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.36.ffn_norm.weight", "offset": 8650973184, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 8650997760, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.36.attn_output.weight", "offset": 8654536704, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.36.attn_q.weight", "offset": 8675770368, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.36.attn_v.weight", "offset": 8697004032, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.37.ffn_gate.weight", "offset": 8702164992, "shape": [ 6144, 16384 ], "size": 5160960, "type": 12 }, { "name": "blk.37.attn_k.weight", "offset": 8758788096, "shape": [ 6144, 1024 ], "size": 56623104, "type": 12 }, { "name": "blk.37.attn_output.weight", "offset": 8762327040, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.37.attn_q.weight", "offset": 8783560704, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.37.attn_v.weight", "offset": 8804794368, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 8808333312, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 8808357888, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 8864980992, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.37.ffn_norm.weight", "offset": 8921604096, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.38.attn_norm.weight", "offset": 8921628672, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 8921653248, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.38.ffn_gate.weight", "offset": 8978276352, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 9034899456, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.38.ffn_norm.weight", "offset": 9091522560, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 9091547136, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.38.attn_output.weight", "offset": 9095086080, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.38.attn_q.weight", "offset": 9116319744, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.38.attn_v.weight", "offset": 9137553408, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 9141092352, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 9141116928, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.39.ffn_gate.weight", "offset": 9223692288, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 9280315392, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.39.ffn_norm.weight", "offset": 9336938496, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 9336963072, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.39.attn_output.weight", "offset": 9340502016, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.39.attn_q.weight", "offset": 9361735680, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.39.attn_v.weight", "offset": 9382969344, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.40.attn_norm.weight", "offset": 9388130304, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 9388154880, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.40.ffn_gate.weight", "offset": 9444777984, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.40.ffn_up.weight", "offset": 9501401088, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.40.ffn_norm.weight", "offset": 9558024192, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 9558048768, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "\u0000blk.55.attn_q.weight\u0002\u0000\u0000\u0000", "offset": 9561587712, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.40.attn_q.weight", "offset": 9582821376, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.40.attn_v.weight", "offset": 9604055040, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.41.attn_norm.weight", "offset": 9607593984, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 9607618560, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.41.ffn_gate.weight", "offset": 9664241664, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.41.ffn_up.weight", "offset": 9720864768, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.41.ffn_norm.weight", "offset": 9777487872, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 9777512448, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.41.attn_output.weight", "offset": 9781051392, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.41.attn_q.weight", "offset": 9802285056, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.41.attn_v.weight", "offset": 9823518720, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.42.attn_norm.weight", "offset": 9827057664, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 9827082240, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.42.ffn_gate.weight", "offset": 9909657600, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.42.ffn_up.weight", "offset": 9966280704, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.42.ffn_norm.weight", "offset": 10022903808, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 10022928384, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.42.attn_output.weight", "offset": 10026467328, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.42.attn_q.weight", "offset": 10047700992, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.42.attn_v.weight", "offset": 10068934656, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.43.ffn_gate.weight", "offset": 10074095616, "shape": [ 6144, 16384 ], "size": 5160960, "type": 12 }, { "name": "blk.43.ffn_up.weight", "offset": 10130718720, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.43.attn_k.weight", "offset": 10187341824, "shape": [ 6144, 1024 ], "size": 56623104, "type": 12 }, { "name": "blk.43.attn_output.weight", "offset": 10190880768, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.43.attn_q.weight", "offset": 10212114432, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.43.attn_v.weight", "offset": 10233348096, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.43.attn_norm.weight", "offset": 10236887040, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 10236911616, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.43.ffn_norm.weight", "offset": 10293534720, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 10293559296, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 10293583872, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.44.ffn_gate.weight", "offset": 10350206976, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.44.ffn_up.weight", "offset": 10406830080, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.44.ffn_norm.weight", "offset": 10463453184, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 10463477760, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.44.attn_output.weight", "offset": 10467016704, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.44.attn_q.weight", "offset": 10488250368, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.44.attn_v.weight", "offset": 10509484032, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.45.attn_norm.weight", "offset": 10513022976, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 10513047552, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.45.ffn_gate.weight", "offset": 10595622912, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.45.ffn_up.weight", "offset": 10652246016, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.45.ffn_norm.weight", "offset": 10708869120, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 10708893696, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.45.attn_output.weight", "offset": 10712432640, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.45.attn_q.weight", "offset": 10733666304, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.45.attn_v.weight", "offset": 10754899968, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.46.attn_norm.weight", "offset": 10760060928, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 10760085504, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.46.ffn_gate.weight", "offset": 10816708608, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.46.ffn_up.weight", "offset": 10873331712, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.46.ffn_norm.weight", "offset": 10929954816, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 10929979392, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.46.attn_output.weight", "offset": 10933518336, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.46.attn_q.weight", "offset": 10954752000, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.46.attn_v.weight", "offset": 10975985664, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.47.attn_norm.weight", "offset": 10979524608, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 10979549184, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.47.ffn_gate.weight", "offset": 11036172288, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.47.ffn_up.weight", "offset": 11092795392, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.47.ffn_norm.weight", "offset": 11149418496, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 11149443072, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.47.attn_output.weight", "offset": 11152982016, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.47.attn_q.weight", "offset": 11174215680, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.47.attn_v.weight", "offset": 11195449344, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.48.attn_norm.weight", "offset": 11198988288, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 11199012864, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.48.ffn_gate.weight", "offset": 11281588224, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.48.ffn_up.weight", "offset": 11338211328, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.48.ffn_norm.weight", "offset": 11394834432, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 11394859008, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.48.attn_output.weight", "offset": 11398397952, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.48.attn_q.weight", "offset": 11419631616, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.48.attn_v.weight", "offset": 11440865280, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.49.attn_norm.weight", "offset": 11446026240, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 11446050816, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.49.ffn_gate.weight", "offset": 11528626176, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.49.ffn_up.weight", "offset": 11585249280, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.49.ffn_norm.weight", "offset": 11641872384, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 11641896960, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.49.attn_output.weight", "offset": 11645435904, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.49.attn_q.weight", "offset": 11666669568, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.49.attn_v.weight", "offset": 11687903232, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.50.attn_k.weight", "offset": 11693064192, "shape": [ 6144, 1024 ], "size": 5160960, "type": 12 }, { "name": "blk.50.attn_q.weight", "offset": 11696603136, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.50.attn_v.weight", "offset": 11717836800, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "output.weight", "offset": 11722997760, "shape": [ 6144, 32768 ], "size": 5160960, "type": 14 }, { "name": "blk.50.attn_norm.weight", "offset": 11888148480, "shape": [ 6144 ], "size": 165150720, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 11888173056, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.50.ffn_gate.weight", "offset": 11970748416, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.50.ffn_up.weight", "offset": 12027371520, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.50.ffn_norm.weight", "offset": 12083994624, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 12084019200, "shape": [ 6144, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.51.attn_norm.weight", "offset": 12105252864, "shape": [ 6144 ], "size": 21233664, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 12105277440, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.51.ffn_gate.weight", "offset": 12187852800, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.51.ffn_up.weight", "offset": 12244475904, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.51.ffn_norm.weight", "offset": 12301099008, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 12301123584, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.51.attn_output.weight", "offset": 12304662528, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.51.attn_q.weight", "offset": 12325896192, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.51.attn_v.weight", "offset": 12347129856, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.52.attn_norm.weight", "offset": 12352290816, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 12352315392, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.52.ffn_gate.weight", "offset": 12434890752, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.52.ffn_up.weight", "offset": 12491513856, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.52.ffn_norm.weight", "offset": 12548136960, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 12548161536, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.52.attn_output.weight", "offset": 12551700480, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.52.attn_q.weight", "offset": 12572934144, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.52.attn_v.weight", "offset": 12594167808, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.53.attn_norm.weight", "offset": 12599328768, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 12599353344, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.53.ffn_gate.weight", "offset": 12681928704, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.53.ffn_up.weight", "offset": 12738551808, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.53.ffn_norm.weight", "offset": 12795174912, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 12795199488, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.53.attn_output.weight", "offset": 12798738432, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.53.attn_q.weight", "offset": 12819972096, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.53.attn_v.weight", "offset": 12841205760, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.54.attn_norm.weight", "offset": 12846366720, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 12846391296, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.54.ffn_gate.weight", "offset": 12928966656, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.54.ffn_up.weight", "offset": 12985589760, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.54.ffn_norm.weight", "offset": 13042212864, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 13042237440, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.54.attn_output.weight", "offset": 13045776384, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.54.attn_q.weight", "offset": 13067010048, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.54.attn_v.weight", "offset": 13088243712, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "blk.55.attn_norm.weight", "offset": 13093404672, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 13093429248, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.55.ffn_gate.weight", "offset": 13176004608, "shape": [ 6144, 16384 ], "size": 82575360, "type": 12 }, { "name": "blk.55.ffn_up.weight", "offset": 13232627712, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.55.ffn_norm.weight", "offset": 13289250816, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 13289275392, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.55.attn_output.weight", "offset": 13292814336, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.55.attn_q.weight", "offset": 13314048000, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.55.attn_v.weight", "offset": 13335281664, "shape": [ 6144, 1024 ], "size": 21233664, "type": 14 }, { "name": "output_norm.weight", "offset": 13340442624, "shape": [ 6144 ], "size": 5160960, "type": 0 } ], "version": 3 }