codestral

{ "metadata": { "general.architecture": "llama", "general.file_type": 14, "general.name": "Codestral-22B-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 32768, "llama.embedding_length": 6144, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_space_prefix": true, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 22247282688, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 113246208, "shape": [ 6144 ], "size": 113246208, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 113270784, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 182476800, "shape": [ 6144, 16384 ], "size": 69206016, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 239099904, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.0.ffn_norm.weight", "offset": 295723008, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 295747584, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 299286528, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 320520192, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.0.attn_v.weight", "offset": 341753856, "shape": [ 6144, 1024 ], "size": 21233664, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 346079232, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 346103808, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 415309824, "shape": [ 6144, 16384 ], "size": 69206016, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 471932928, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 528556032, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 528580608, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 532119552, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 553353216, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.1.attn_v.weight", "offset": 574586880, "shape": [ 6144, 1024 ], "size": 21233664, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 578912256, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 578936832, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 648142848, "shape": [ 6144, 16384 ], "size": 69206016, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 704765952, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 761389056, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 761413632, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 764952576, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 786186240, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.2.attn_v.weight", "offset": 807419904, "shape": [ 6144, 1024 ], "size": 21233664, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 811745280, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 811769856, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 880975872, "shape": [ 6144, 16384 ], "size": 69206016, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 937598976, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 994222080, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 994246656, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 997785600, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 1019019264, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.3.attn_v.weight", "offset": 1040252928, "shape": [ 6144, 1024 ], "size": 21233664, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 1044578304, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1044602880, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 1113808896, "shape": [ 6144, 16384 ], "size": 69206016, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 1170432000, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 1227055104, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1227079680, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 1230618624, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 1251852288, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.4.attn_v.weight", "offset": 1273085952, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 1276624896, "shape": [ 6144, 16384 ], "size": 3538944, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 1333248000, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.5.attn_k.weight", "offset": 1389871104, "shape": [ 6144, 1024 ], "size": 56623104, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 1393410048, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 1414643712, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.5.attn_v.weight", "offset": 1435877376, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1439416320, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1439440896, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 1508646912, "shape": [ 6144, 16384 ], "size": 69206016, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 1565270016, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 1621893120, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1621917696, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 1625456640, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.10.attn_q.weight", "offset": 1646690304, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.10.attn_v.weight", "offset": 1667923968, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1671462912, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1671487488, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 1740693504, "shape": [ 6144, 16384 ], "size": 69206016, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 1797316608, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 1853939712, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1853964288, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 1857503232, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.11.attn_q.weight", "offset": 1878736896, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.11.attn_v.weight", "offset": 1899970560, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.12.attn_k.weight", "offset": 1903509504, "shape": [ 6144, 1024 ], "size": 3538944, "type": 12 }, { "name": "blk.12.attn_q.weight", "offset": 1907048448, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.12.attn_v.weight", "offset": 1928282112, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 1931821056, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1931845632, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 1988468736, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 1988493312, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1988517888, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 2045140992, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 2101764096, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 2158387200, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 2158411776, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 2161950720, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 2183184384, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.6.attn_v.weight", "offset": 2204418048, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 2207956992, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2207981568, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 2264604672, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 2321227776, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 2377850880, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2377875456, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 2381414400, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 2402648064, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.7.attn_v.weight", "offset": 2423881728, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 2427420672, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2427445248, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 2484068352, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 2540691456, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 2597314560, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2597339136, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 2600878080, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 2622111744, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.8.attn_v.weight", "offset": 2643345408, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 2646884352, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2646908928, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 2703532032, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 2760155136, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.9.ffn_norm.weight", "offset": 2816778240, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2816802816, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 2820341760, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 2841575424, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.9.attn_v.weight", "offset": 2862809088, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 2866348032, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2866372608, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 2922995712, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 2979618816, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 3036241920, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 3036266496, "shape": [ 6144, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 3057500160, "shape": [ 6144 ], "size": 21233664, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 3057524736, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 3114147840, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 3170770944, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 3227394048, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 3227418624, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 3230957568, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.13.attn_q.weight", "offset": 3252191232, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.13.attn_v.weight", "offset": 3273424896, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 3276963840, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3276988416, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 3333611520, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 3390234624, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 3446857728, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 3446882304, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 3450421248, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 3471654912, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.14.attn_v.weight", "offset": 3492888576, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 3496427520, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3496452096, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 3553075200, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 3609698304, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 3666321408, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 3666345984, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 3669884928, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 3691118592, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.15.attn_v.weight", "offset": 3712352256, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 3715891200, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3715915776, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 3772538880, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 3829161984, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 3885785088, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 3885809664, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 3889348608, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.16.attn_q.weight", "offset": 3910582272, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.16.attn_v.weight", "offset": 3931815936, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 3935354880, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3935379456, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 3992002560, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 4048625664, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 4105248768, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 4105273344, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 4108812288, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 4130045952, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.17.attn_v.weight", "offset": 4151279616, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 4154818560, "shape": [ 6144, 16384 ], "size": 3538944, "type": 12 }, { "name": "blk.18.attn_k.weight", "offset": 4211441664, "shape": [ 6144, 1024 ], "size": 56623104, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 4214980608, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.18.attn_q.weight", "offset": 4236214272, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.18.attn_v.weight", "offset": 4257447936, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 4260986880, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 4261011456, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 4317634560, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 4374257664, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 4374282240, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4374306816, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 4430929920, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 4487553024, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.19.ffn_norm.weight", "offset": 4544176128, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 4544200704, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 4547739648, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 4568973312, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.19.attn_v.weight", "offset": 4590206976, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 4593745920, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4593770496, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 4650393600, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 4707016704, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 4763639808, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 4763664384, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 4767203328, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 4788436992, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.20.attn_v.weight", "offset": 4809670656, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 4813209600, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4813234176, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 4869857280, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 4926480384, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 4983103488, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 4983128064, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 4986667008, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 5007900672, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.21.attn_v.weight", "offset": 5029134336, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 5032673280, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5032697856, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 5089320960, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 5145944064, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 5202567168, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 5202591744, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 5206130688, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 5227364352, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.22.attn_v.weight", "offset": 5248598016, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 5252136960, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5252161536, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 5308784640, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 5365407744, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 5422030848, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 5422055424, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 5425594368, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 5446828032, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.23.attn_v.weight", "offset": 5468061696, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 5471600640, "shape": [ 6144, 16384 ], "size": 3538944, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 5528223744, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.24.attn_k.weight", "offset": 5584846848, "shape": [ 6144, 1024 ], "size": 56623104, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 5588385792, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 5609619456, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.24.attn_v.weight", "offset": 5630853120, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 5634392064, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5634416640, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 5691039744, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.25.attn_norm.weight", "offset": 5691064320, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5691088896, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 5747712000, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 5804335104, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 5860958208, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 5860982784, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 5864521728, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 5885755392, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.25.attn_v.weight", "offset": 5906989056, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 5910528000, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5910552576, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 5967175680, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 6023798784, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 6080421888, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 6080446464, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 6083985408, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 6105219072, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.26.attn_v.weight", "offset": 6126452736, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 6129991680, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 6130016256, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 6186639360, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 6243262464, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.27.ffn_norm.weight", "offset": 6299885568, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 6299910144, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 6303449088, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.27.attn_q.weight", "offset": 6324682752, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.27.attn_v.weight", "offset": 6345916416, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 6349455360, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 6349479936, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.28.ffn_gate.weight", "offset": 6406103040, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 6462726144, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.28.ffn_norm.weight", "offset": 6519349248, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 6519373824, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 6522912768, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.28.attn_q.weight", "offset": 6544146432, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.28.attn_v.weight", "offset": 6565380096, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 6568919040, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 6568943616, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.29.ffn_gate.weight", "offset": 6625566720, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 6682189824, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.29.ffn_norm.weight", "offset": 6738812928, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 6738837504, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 6742376448, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.29.attn_q.weight", "offset": 6763610112, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.29.attn_v.weight", "offset": 6784843776, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 6788382720, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 6788407296, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.30.ffn_gate.weight", "offset": 6845030400, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 6901653504, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.30.ffn_norm.weight", "offset": 6958276608, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 6958301184, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 6961840128, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.30.attn_q.weight", "offset": 6983073792, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.30.attn_v.weight", "offset": 7004307456, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.31.attn_k.weight", "offset": 7007846400, "shape": [ 6144, 1024 ], "size": 3538944, "type": 12 }, { "name": "blk.31.attn_q.weight", "offset": 7011385344, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.31.attn_v.weight", "offset": 7032619008, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 7036157952, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 7036182528, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.31.ffn_gate.weight", "offset": 7092805632, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 7149428736, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.31.ffn_norm.weight", "offset": 7206051840, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 7206076416, "shape": [ 6144, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 7227310080, "shape": [ 6144 ], "size": 21233664, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 7227334656, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.32.ffn_gate.weight", "offset": 7283957760, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 7340580864, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.32.ffn_norm.weight", "offset": 7397203968, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 7397228544, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.32.attn_output.weight", "offset": 7400767488, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.32.attn_q.weight", "offset": 7422001152, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.32.attn_v.weight", "offset": 7443234816, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 7446773760, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 7446798336, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.33.ffn_gate.weight", "offset": 7503421440, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 7560044544, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.33.ffn_norm.weight", "offset": 7616667648, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 7616692224, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.33.attn_output.weight", "offset": 7620231168, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.33.attn_q.weight", "offset": 7641464832, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.33.attn_v.weight", "offset": 7662698496, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 7666237440, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 7666262016, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.34.ffn_gate.weight", "offset": 7722885120, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.34.ffn_up.weight", "offset": 7779508224, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.34.ffn_norm.weight", "offset": 7836131328, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 7836155904, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.34.attn_output.weight", "offset": 7839694848, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.34.attn_q.weight", "offset": 7860928512, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.34.attn_v.weight", "offset": 7882162176, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.35.attn_norm.weight", "offset": 7885701120, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 7885725696, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.35.ffn_gate.weight", "offset": 7942348800, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 7998971904, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.35.ffn_norm.weight", "offset": 8055595008, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 8055619584, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 8059158528, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.35.attn_q.weight", "offset": 8080392192, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.35.attn_v.weight", "offset": 8101625856, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.36.attn_norm.weight", "offset": 8105164800, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 8105189376, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.36.ffn_gate.weight", "offset": 8161812480, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 8218435584, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.36.ffn_norm.weight", "offset": 8275058688, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 8275083264, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.36.attn_output.weight", "offset": 8278622208, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.36.attn_q.weight", "offset": 8299855872, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.36.attn_v.weight", "offset": 8321089536, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.37.ffn_gate.weight", "offset": 8324628480, "shape": [ 6144, 16384 ], "size": 3538944, "type": 12 }, { "name": "blk.37.attn_k.weight", "offset": 8381251584, "shape": [ 6144, 1024 ], "size": 56623104, "type": 12 }, { "name": "blk.37.attn_output.weight", "offset": 8384790528, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.37.attn_q.weight", "offset": 8406024192, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.37.attn_v.weight", "offset": 8427257856, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 8430796800, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 8430821376, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 8487444480, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.37.ffn_norm.weight", "offset": 8544067584, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.38.attn_norm.weight", "offset": 8544092160, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 8544116736, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.38.ffn_gate.weight", "offset": 8600739840, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 8657362944, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.38.ffn_norm.weight", "offset": 8713986048, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 8714010624, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.38.attn_output.weight", "offset": 8717549568, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.38.attn_q.weight", "offset": 8738783232, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.38.attn_v.weight", "offset": 8760016896, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 8763555840, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 8763580416, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.39.ffn_gate.weight", "offset": 8820203520, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 8876826624, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.39.ffn_norm.weight", "offset": 8933449728, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 8933474304, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.39.attn_output.weight", "offset": 8937013248, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.39.attn_q.weight", "offset": 8958246912, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.39.attn_v.weight", "offset": 8979480576, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.40.attn_norm.weight", "offset": 8983019520, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 8983044096, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.40.ffn_gate.weight", "offset": 9039667200, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.40.ffn_up.weight", "offset": 9096290304, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.40.ffn_norm.weight", "offset": 9152913408, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 9152937984, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.40.attn_output.weight", "offset": 9156476928, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.40.attn_q.weight", "offset": 9177710592, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.40.attn_v.weight", "offset": 9198944256, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.41.attn_norm.weight", "offset": 9202483200, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 9202507776, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.41.ffn_gate.weight", "offset": 9259130880, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.41.ffn_up.weight", "offset": 9315753984, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.41.ffn_norm.weight", "offset": 9372377088, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 9372401664, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.41.attn_output.weight", "offset": 9375940608, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.41.attn_q.weight", "offset": 9397174272, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.41.attn_v.weight", "offset": 9418407936, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.42.attn_norm.weight", "offset": 9421946880, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 9421971456, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.42.ffn_gate.weight", "offset": 9478594560, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.42.ffn_up.weight", "offset": 9535217664, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.42.ffn_norm.weight", "offset": 9591840768, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 9591865344, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.42.attn_output.weight", "offset": 9595404288, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.42.attn_q.weight", "offset": 9616637952, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.42.attn_v.weight", "offset": 9637871616, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.43.ffn_gate.weight", "offset": 9641410560, "shape": [ 6144, 16384 ], "size": 3538944, "type": 12 }, { "name": "blk.43.ffn_up.weight", "offset": 9698033664, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.43.attn_k.weight", "offset": 9754656768, "shape": [ 6144, 1024 ], "size": 56623104, "type": 12 }, { "name": "blk.43.attn_output.weight", "offset": 9758195712, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.43.attn_q.weight", "offset": 9779429376, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.43.attn_v.weight", "offset": 9800663040, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.43.attn_norm.weight", "offset": 9804201984, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 9804226560, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.43.ffn_norm.weight", "offset": 9860849664, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 9860874240, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 9860898816, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.44.ffn_gate.weight", "offset": 9917521920, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.44.ffn_up.weight", "offset": 9974145024, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.44.ffn_norm.weight", "offset": 10030768128, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 10030792704, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.44.attn_output.weight", "offset": 10034331648, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.44.attn_q.weight", "offset": 10055565312, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.44.attn_v.weight", "offset": 10076798976, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.45.attn_norm.weight", "offset": 10080337920, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 10080362496, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.45.ffn_gate.weight", "offset": 10136985600, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.45.ffn_up.weight", "offset": 10193608704, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.45.ffn_norm.weight", "offset": 10250231808, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 10250256384, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.45.attn_output.weight", "offset": 10253795328, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.45.attn_q.weight", "offset": 10275028992, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.45.attn_v.weight", "offset": 10296262656, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.46.attn_norm.weight", "offset": 10299801600, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 10299826176, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.46.ffn_gate.weight", "offset": 10356449280, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.46.ffn_up.weight", "offset": 10413072384, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.46.ffn_norm.weight", "offset": 10469695488, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 10469720064, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.46.attn_output.weight", "offset": 10473259008, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.46.attn_q.weight", "offset": 10494492672, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.46.attn_v.weight", "offset": 10515726336, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.47.attn_norm.weight", "offset": 10519265280, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 10519289856, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.47.ffn_gate.weight", "offset": 10575912960, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.47.ffn_up.weight", "offset": 10632536064, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.47.ffn_norm.weight", "offset": 10689159168, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 10689183744, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.47.attn_output.weight", "offset": 10692722688, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.47.attn_q.weight", "offset": 10713956352, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.47.attn_v.weight", "offset": 10735190016, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.48.attn_norm.weight", "offset": 10738728960, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 10738753536, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.48.ffn_gate.weight", "offset": 10795376640, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.48.ffn_up.weight", "offset": 10851999744, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.48.ffn_norm.weight", "offset": 10908622848, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 10908647424, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.48.attn_output.weight", "offset": 10912186368, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.48.attn_q.weight", "offset": 10933420032, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.48.attn_v.weight", "offset": 10954653696, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.49.attn_norm.weight", "offset": 10958192640, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 10958217216, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.49.ffn_gate.weight", "offset": 11014840320, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.49.ffn_up.weight", "offset": 11071463424, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.49.ffn_norm.weight", "offset": 11128086528, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 11128111104, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.49.attn_output.weight", "offset": 11131650048, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.49.attn_q.weight", "offset": 11152883712, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.49.attn_v.weight", "offset": 11174117376, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.50.attn_k.weight", "offset": 11177656320, "shape": [ 6144, 1024 ], "size": 3538944, "type": 12 }, { "name": "blk.50.attn_q.weight", "offset": 11181195264, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.50.attn_v.weight", "offset": 11202428928, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "output.weight", "offset": 11205967872, "shape": [ 6144, 32768 ], "size": 3538944, "type": 14 }, { "name": "blk.50.attn_norm.weight", "offset": 11371118592, "shape": [ 6144 ], "size": 165150720, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 11371143168, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.50.ffn_gate.weight", "offset": 11427766272, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.50.ffn_up.weight", "offset": 11484389376, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.50.ffn_norm.weight", "offset": 11541012480, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 11541037056, "shape": [ 6144, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.51.attn_norm.weight", "offset": 11562270720, "shape": [ 6144 ], "size": 21233664, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 11562295296, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.51.ffn_gate.weight", "offset": 11618918400, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.51.ffn_up.weight", "offset": 11675541504, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.51.ffn_norm.weight", "offset": 11732164608, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 11732189184, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.51.attn_output.weight", "offset": 11735728128, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.51.attn_q.weight", "offset": 11756961792, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.51.attn_v.weight", "offset": 11778195456, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.52.attn_norm.weight", "offset": 11781734400, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 11781758976, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.52.ffn_gate.weight", "offset": 11838382080, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.52.ffn_up.weight", "offset": 11895005184, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.52.ffn_norm.weight", "offset": 11951628288, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 11951652864, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.52.attn_output.weight", "offset": 11955191808, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.52.attn_q.weight", "offset": 11976425472, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.52.attn_v.weight", "offset": 11997659136, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.53.attn_norm.weight", "offset": 12001198080, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 12001222656, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.53.ffn_gate.weight", "offset": 12057845760, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.53.ffn_up.weight", "offset": 12114468864, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.53.ffn_norm.weight", "offset": 12171091968, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 12171116544, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.53.attn_output.weight", "offset": 12174655488, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.53.attn_q.weight", "offset": 12195889152, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.53.attn_v.weight", "offset": 12217122816, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.54.attn_norm.weight", "offset": 12220661760, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 12220686336, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.54.ffn_gate.weight", "offset": 12277309440, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.54.ffn_up.weight", "offset": 12333932544, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.54.ffn_norm.weight", "offset": 12390555648, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 12390580224, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.54.attn_output.weight", "offset": 12394119168, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.54.attn_q.weight", "offset": 12415352832, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.54.attn_v.weight", "offset": 12436586496, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "blk.55.attn_norm.weight", "offset": 12440125440, "shape": [ 6144 ], "size": 3538944, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 12440150016, "shape": [ 16384, 6144 ], "size": 24576, "type": 12 }, { "name": "blk.55.ffn_gate.weight", "offset": 12496773120, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.55.ffn_up.weight", "offset": 12553396224, "shape": [ 6144, 16384 ], "size": 56623104, "type": 12 }, { "name": "blk.55.ffn_norm.weight", "offset": 12610019328, "shape": [ 6144 ], "size": 56623104, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 12610043904, "shape": [ 6144, 1024 ], "size": 24576, "type": 12 }, { "name": "blk.55.attn_output.weight", "offset": 12613582848, "shape": [ 6144, 6144 ], "size": 3538944, "type": 12 }, { "name": "blk.55.attn_q.weight", "offset": 12634816512, "shape": [ 6144, 6144 ], "size": 21233664, "type": 12 }, { "name": "blk.55.attn_v.weight", "offset": 12656050176, "shape": [ 6144, 1024 ], "size": 21233664, "type": 12 }, { "name": "output_norm.weight", "offset": 12659589120, "shape": [ 6144 ], "size": 3538944, "type": 0 } ], "version": 3 }