codestral

{ "metadata": { "general.architecture": "llama", "general.file_type": 3, "general.name": "Codestral-22B-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 32768, "llama.embedding_length": 6144, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_space_prefix": true, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 22247282688, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 125829120, "shape": [ 6144 ], "size": 125829120, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 125853696, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.0.ffn_gate.weight", "offset": 188768256, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.0.ffn_up.weight", "offset": 251682816, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.0.ffn_norm.weight", "offset": 314597376, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 314621952, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.0.attn_output.weight", "offset": 318554112, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.0.attn_q.weight", "offset": 342147072, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.0.attn_v.weight", "offset": 365740032, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.1.attn_norm.weight", "offset": 369672192, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 369696768, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.1.ffn_gate.weight", "offset": 432611328, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.1.ffn_up.weight", "offset": 495525888, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.1.ffn_norm.weight", "offset": 558440448, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 558465024, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.1.attn_output.weight", "offset": 562397184, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.1.attn_q.weight", "offset": 585990144, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.1.attn_v.weight", "offset": 609583104, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.2.attn_norm.weight", "offset": 613515264, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 613539840, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.2.ffn_gate.weight", "offset": 676454400, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.2.ffn_up.weight", "offset": 739368960, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.2.ffn_norm.weight", "offset": 802283520, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "lk.17.attn_v.weight", "offset": 802308096, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.2.attn_output.weight", "offset": 806240256, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.2.attn_q.weight", "offset": 829833216, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.2.attn_v.weight", "offset": 853426176, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.3.attn_norm.weight", "offset": 857358336, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 857382912, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.3.ffn_gate.weight", "offset": 920297472, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.3.ffn_up.weight", "offset": 983212032, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.3.ffn_norm.weight", "offset": 1046126592, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1046151168, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.3.attn_output.weight", "offset": 1050083328, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.3.attn_q.weight", "offset": 1073676288, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.3.attn_v.weight", "offset": 1097269248, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.4.attn_norm.weight", "offset": 1101201408, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1101225984, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.4.ffn_gate.weight", "offset": 1164140544, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.4.ffn_up.weight", "offset": 1227055104, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.4.ffn_norm.weight", "offset": 1289969664, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1289994240, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.4.attn_output.weight", "offset": 1293926400, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.4.attn_q.weight", "offset": 1317519360, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.4.attn_v.weight", "offset": 1341112320, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.5.ffn_gate.weight", "offset": 1345044480, "shape": [ 6144, 16384 ], "size": 3932160, "type": 3 }, { "name": "blk.5.ffn_up.weight", "offset": 1407959040, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.5.attn_k.weight", "offset": 1470873600, "shape": [ 6144, 1024 ], "size": 62914560, "type": 3 }, { "name": "blk.5.attn_output.weight", "offset": 1474805760, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.5.attn_q.weight", "offset": 1498398720, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.5.attn_v.weight", "offset": 1521991680, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.10.attn_norm.weight", "offset": 1525923840, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1525948416, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.10.ffn_gate.weight", "offset": 1588862976, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.10.ffn_up.weight", "offset": 1651777536, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.10.ffn_norm.weight", "offset": 1714692096, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1714716672, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.10.attn_output.weight", "offset": 1718648832, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.10.attn_q.weight", "offset": 1742241792, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.10.attn_v.weight", "offset": 1765834752, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.11.attn_norm.weight", "offset": 1769766912, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1769791488, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.11.ffn_gate.weight", "offset": 1832706048, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.11.ffn_up.weight", "offset": 1895620608, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.11.ffn_norm.weight", "offset": 1958535168, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1958559744, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.11.attn_output.weight", "offset": 1962491904, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.11.attn_q.weight", "offset": 1986084864, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.11.attn_v.weight", "offset": 2009677824, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.12.attn_k.weight", "offset": 2013609984, "shape": [ 6144, 1024 ], "size": 3932160, "type": 3 }, { "name": "blk.12.attn_q.weight", "offset": 2017542144, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.12.attn_v.weight", "offset": 2041135104, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.5.attn_norm.weight", "offset": 2045067264, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 2045091840, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.5.ffn_norm.weight", "offset": 2108006400, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 2108030976, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2108055552, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.6.ffn_gate.weight", "offset": 2170970112, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.6.ffn_up.weight", "offset": 2233884672, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.6.ffn_norm.weight", "offset": 2296799232, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 2296823808, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.6.attn_output.weight", "offset": 2300755968, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.6.attn_q.weight", "offset": 2324348928, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.6.attn_v.weight", "offset": 2347941888, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.7.attn_norm.weight", "offset": 2351874048, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2351898624, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.7.ffn_gate.weight", "offset": 2414813184, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.7.ffn_up.weight", "offset": 2477727744, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.7.ffn_norm.weight", "offset": 2540642304, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2540666880, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.7.attn_output.weight", "offset": 2544599040, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.7.attn_q.weight", "offset": 2568192000, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.7.attn_v.weight", "offset": 2591784960, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.8.attn_norm.weight", "offset": 2595717120, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2595741696, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.8.ffn_gate.weight", "offset": 2658656256, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.8.ffn_up.weight", "offset": 2721570816, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.8.ffn_norm.weight", "offset": 2784485376, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2784509952, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.8.attn_output.weight", "offset": 2788442112, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.8.attn_q.weight", "offset": 2812035072, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.8.attn_v.weight", "offset": 2835628032, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.9.attn_norm.weight", "offset": 2839560192, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2839584768, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.9.ffn_gate.weight", "offset": 2902499328, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.9.ffn_up.weight", "offset": 2965413888, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.9.ffn_norm.weight", "offset": 3028328448, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 3028353024, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.9.attn_output.weight", "offset": 3032285184, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.9.attn_q.weight", "offset": 3055878144, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.9.attn_v.weight", "offset": 3079471104, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.12.attn_norm.weight", "offset": 3083403264, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 3083427840, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.12.ffn_gate.weight", "offset": 3146342400, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.12.ffn_up.weight", "offset": 3209256960, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.12.ffn_norm.weight", "offset": 3272171520, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 3272196096, "shape": [ 6144, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.13.attn_norm.weight", "offset": 3295789056, "shape": [ 6144 ], "size": 23592960, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 3295813632, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.13.ffn_gate.weight", "offset": 3358728192, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.13.ffn_up.weight", "offset": 3421642752, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.13.ffn_norm.weight", "offset": 3484557312, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 3484581888, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.13.attn_output.weight", "offset": 3488514048, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.13.attn_q.weight", "offset": 3512107008, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.13.attn_v.weight", "offset": 3535699968, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.14.attn_norm.weight", "offset": 3539632128, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3539656704, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.14.ffn_gate.weight", "offset": 3602571264, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.14.ffn_up.weight", "offset": 3665485824, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.14.ffn_norm.weight", "offset": 3728400384, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 3728424960, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.14.attn_output.weight", "offset": 3732357120, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.14.attn_q.weight", "offset": 3755950080, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.14.attn_v.weight", "offset": 3779543040, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.15.attn_norm.weight", "offset": 3783475200, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3783499776, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.15.ffn_gate.weight", "offset": 3846414336, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.15.ffn_up.weight", "offset": 3909328896, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.15.ffn_norm.weight", "offset": 3972243456, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 3972268032, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.15.attn_output.weight", "offset": 3976200192, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.15.attn_q.weight", "offset": 3999793152, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.15.attn_v.weight", "offset": 4023386112, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.16.attn_norm.weight", "offset": 4027318272, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 4027342848, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.16.ffn_gate.weight", "offset": 4090257408, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.16.ffn_up.weight", "offset": 4153171968, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.16.ffn_norm.weight", "offset": 4216086528, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 4216111104, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.16.attn_output.weight", "offset": 4220043264, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.16.attn_q.weight", "offset": 4243636224, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.16.attn_v.weight", "offset": 4267229184, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.17.attn_norm.weight", "offset": 4271161344, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4271185920, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.17.ffn_gate.weight", "offset": 4334100480, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.17.ffn_up.weight", "offset": 4397015040, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.17.ffn_norm.weight", "offset": 4459929600, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 4459954176, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.17.attn_output.weight", "offset": 4463886336, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.17.attn_q.weight", "offset": 4487479296, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "33.ffn_down.weight\u0002\u0000", "offset": 4511072256, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.18.ffn_gate.weight", "offset": 4515004416, "shape": [ 6144, 16384 ], "size": 3932160, "type": 3 }, { "name": "blk.18.attn_k.weight", "offset": 4577918976, "shape": [ 6144, 1024 ], "size": 62914560, "type": 3 }, { "name": "blk.18.attn_output.weight", "offset": 4581851136, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.18.attn_q.weight", "offset": 4605444096, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.18.attn_v.weight", "offset": 4629037056, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.18.attn_norm.weight", "offset": 4632969216, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 4632993792, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.18.ffn_up.weight", "offset": 4695908352, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.18.ffn_norm.weight", "offset": 4758822912, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 4758847488, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4758872064, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.19.ffn_gate.weight", "offset": 4821786624, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.19.ffn_up.weight", "offset": 4884701184, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.19.ffn_norm.weight", "offset": 4947615744, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 4947640320, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.19.attn_output.weight", "offset": 4951572480, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.19.attn_q.weight", "offset": 4975165440, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.19.attn_v.weight", "offset": 4998758400, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.20.attn_norm.weight", "offset": 5002690560, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 5002715136, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.20.ffn_gate.weight", "offset": 5065629696, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.20.ffn_up.weight", "offset": 5128544256, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.20.ffn_norm.weight", "offset": 5191458816, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 5191483392, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.20.attn_output.weight", "offset": 5195415552, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.20.attn_q.weight", "offset": 5219008512, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.20.attn_v.weight", "offset": 5242601472, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.21.attn_norm.weight", "offset": 5246533632, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5246558208, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.21.ffn_gate.weight", "offset": 5309472768, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.21.ffn_up.weight", "offset": 5372387328, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.21.ffn_norm.weight", "offset": 5435301888, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 5435326464, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.21.attn_output.weight", "offset": 5439258624, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.21.attn_q.weight", "offset": 5462851584, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.21.attn_v.weight", "offset": 5486444544, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.22.attn_norm.weight", "offset": 5490376704, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5490401280, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.22.ffn_gate.weight", "offset": 5553315840, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.22.ffn_up.weight", "offset": 5616230400, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.22.ffn_norm.weight", "offset": 5679144960, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 5679169536, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.22.attn_output.weight", "offset": 5683101696, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.22.attn_q.weight", "offset": 5706694656, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.22.attn_v.weight", "offset": 5730287616, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.23.attn_norm.weight", "offset": 5734219776, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5734244352, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.23.ffn_gate.weight", "offset": 5797158912, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.23.ffn_up.weight", "offset": 5860073472, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.23.ffn_norm.weight", "offset": 5922988032, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 5923012608, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.23.attn_output.weight", "offset": 5926944768, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.23.attn_q.weight", "offset": 5950537728, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.23.attn_v.weight", "offset": 5974130688, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.24.ffn_gate.weight", "offset": 5978062848, "shape": [ 6144, 16384 ], "size": 3932160, "type": 3 }, { "name": "blk.24.ffn_up.weight", "offset": 6040977408, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.24.attn_k.weight", "offset": 6103891968, "shape": [ 6144, 1024 ], "size": 62914560, "type": 3 }, { "name": "blk.24.attn_output.weight", "offset": 6107824128, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.24.attn_q.weight", "offset": 6131417088, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.24.attn_v.weight", "offset": 6155010048, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.24.attn_norm.weight", "offset": 6158942208, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 6158966784, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.24.ffn_norm.weight", "offset": 6221881344, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.25.attn_norm.weight", "offset": 6221905920, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 6221930496, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.25.ffn_gate.weight", "offset": 6284845056, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.25.ffn_up.weight", "offset": 6347759616, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.25.ffn_norm.weight", "offset": 6410674176, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 6410698752, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.25.attn_output.weight", "offset": 6414630912, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.25.attn_q.weight", "offset": 6438223872, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.25.attn_v.weight", "offset": 6461816832, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.26.attn_norm.weight", "offset": 6465748992, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 6465773568, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.26.ffn_gate.weight", "offset": 6528688128, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.26.ffn_up.weight", "offset": 6591602688, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.26.ffn_norm.weight", "offset": 6654517248, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 6654541824, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.26.attn_output.weight", "offset": 6658473984, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.26.attn_q.weight", "offset": 6682066944, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.26.attn_v.weight", "offset": 6705659904, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.27.attn_norm.weight", "offset": 6709592064, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 6709616640, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.27.ffn_gate.weight", "offset": 6772531200, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.27.ffn_up.weight", "offset": 6835445760, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.27.ffn_norm.weight", "offset": 6898360320, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 6898384896, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.27.attn_output.weight", "offset": 6902317056, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.27.attn_q.weight", "offset": 6925910016, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.27.attn_v.weight", "offset": 6949502976, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.28.attn_norm.weight", "offset": 6953435136, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 6953459712, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.28.ffn_gate.weight", "offset": 7016374272, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.28.ffn_up.weight", "offset": 7079288832, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.28.ffn_norm.weight", "offset": 7142203392, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 7142227968, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.28.attn_output.weight", "offset": 7146160128, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.28.attn_q.weight", "offset": 7169753088, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.28.attn_v.weight", "offset": 7193346048, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.29.attn_norm.weight", "offset": 7197278208, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 7197302784, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.29.ffn_gate.weight", "offset": 7260217344, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.29.ffn_up.weight", "offset": 7323131904, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.29.ffn_norm.weight", "offset": 7386046464, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 7386071040, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.29.attn_output.weight", "offset": 7390003200, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.29.attn_q.weight", "offset": 7413596160, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.29.attn_v.weight", "offset": 7437189120, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.30.attn_norm.weight", "offset": 7441121280, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 7441145856, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.30.ffn_gate.weight", "offset": 7504060416, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.30.ffn_up.weight", "offset": 7566974976, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.30.ffn_norm.weight", "offset": 7629889536, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 7629914112, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.30.attn_output.weight", "offset": 7633846272, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.30.attn_q.weight", "offset": 7657439232, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.30.attn_v.weight", "offset": 7681032192, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.31.attn_k.weight", "offset": 7684964352, "shape": [ 6144, 1024 ], "size": 3932160, "type": 3 }, { "name": "blk.31.attn_q.weight", "offset": 7688896512, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.31.attn_v.weight", "offset": 7712489472, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.31.attn_norm.weight", "offset": 7716421632, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 7716446208, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.31.ffn_gate.weight", "offset": 7779360768, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.31.ffn_up.weight", "offset": 7842275328, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.31.ffn_norm.weight", "offset": 7905189888, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 7905214464, "shape": [ 6144, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.32.attn_norm.weight", "offset": 7928807424, "shape": [ 6144 ], "size": 23592960, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 7928832000, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.32.ffn_gate.weight", "offset": 7991746560, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.32.ffn_up.weight", "offset": 8054661120, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.32.ffn_norm.weight", "offset": 8117575680, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 8117600256, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.32.attn_output.weight", "offset": 8121532416, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.32.attn_q.weight", "offset": 8145125376, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.32.attn_v.weight", "offset": 8168718336, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.33.attn_norm.weight", "offset": 8172650496, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 8172675072, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.33.ffn_gate.weight", "offset": 8235589632, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.33.ffn_up.weight", "offset": 8298504192, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.33.ffn_norm.weight", "offset": 8361418752, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 8361443328, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.33.attn_output.weight", "offset": 8365375488, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.33.attn_q.weight", "offset": 8388968448, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.33.attn_v.weight", "offset": 8412561408, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.34.attn_norm.weight", "offset": 8416493568, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 8416518144, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.34.ffn_gate.weight", "offset": 8479432704, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.34.ffn_up.weight", "offset": 8542347264, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.34.ffn_norm.weight", "offset": 8605261824, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 8605286400, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.34.attn_output.weight", "offset": 8609218560, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.34.attn_q.weight", "offset": 8632811520, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.34.attn_v.weight", "offset": 8656404480, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.35.attn_norm.weight", "offset": 8660336640, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 8660361216, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.35.ffn_gate.weight", "offset": 8723275776, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.35.ffn_up.weight", "offset": 8786190336, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.35.ffn_norm.weight", "offset": 8849104896, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 8849129472, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.35.attn_output.weight", "offset": 8853061632, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.35.attn_q.weight", "offset": 8876654592, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.35.attn_v.weight", "offset": 8900247552, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.36.attn_norm.weight", "offset": 8904179712, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 8904204288, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.36.ffn_gate.weight", "offset": 8967118848, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.36.ffn_up.weight", "offset": 9030033408, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.36.ffn_norm.weight", "offset": 9092947968, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 9092972544, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.36.attn_output.weight", "offset": 9096904704, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.36.attn_q.weight", "offset": 9120497664, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.36.attn_v.weight", "offset": 9144090624, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.37.ffn_gate.weight", "offset": 9148022784, "shape": [ 6144, 16384 ], "size": 3932160, "type": 3 }, { "name": "blk.37.attn_k.weight", "offset": 9210937344, "shape": [ 6144, 1024 ], "size": 62914560, "type": 3 }, { "name": "blk.37.attn_output.weight", "offset": 9214869504, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.37.attn_q.weight", "offset": 9238462464, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.37.attn_v.weight", "offset": 9262055424, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.37.attn_norm.weight", "offset": 9265987584, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 9266012160, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.37.ffn_up.weight", "offset": 9328926720, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.37.ffn_norm.weight", "offset": 9391841280, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.38.attn_norm.weight", "offset": 9391865856, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 9391890432, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.38.ffn_gate.weight", "offset": 9454804992, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.38.ffn_up.weight", "offset": 9517719552, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.38.ffn_norm.weight", "offset": 9580634112, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 9580658688, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.38.attn_output.weight", "offset": 9584590848, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.38.attn_q.weight", "offset": 9608183808, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.38.attn_v.weight", "offset": 9631776768, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.39.attn_norm.weight", "offset": 9635708928, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 9635733504, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.39.ffn_gate.weight", "offset": 9698648064, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.39.ffn_up.weight", "offset": 9761562624, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.39.ffn_norm.weight", "offset": 9824477184, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 9824501760, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.39.attn_output.weight", "offset": 9828433920, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.39.attn_q.weight", "offset": 9852026880, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.39.attn_v.weight", "offset": 9875619840, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.40.attn_norm.weight", "offset": 9879552000, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 9879576576, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.40.ffn_gate.weight", "offset": 9942491136, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.40.ffn_up.weight", "offset": 10005405696, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.40.ffn_norm.weight", "offset": 10068320256, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 10068344832, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.40.attn_output.weight", "offset": 10072276992, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.40.attn_q.weight", "offset": 10095869952, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.40.attn_v.weight", "offset": 10119462912, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.41.attn_norm.weight", "offset": 10123395072, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 10123419648, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.41.ffn_gate.weight", "offset": 10186334208, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.41.ffn_up.weight", "offset": 10249248768, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.41.ffn_norm.weight", "offset": 10312163328, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 10312187904, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.41.attn_output.weight", "offset": 10316120064, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.41.attn_q.weight", "offset": 10339713024, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.41.attn_v.weight", "offset": 10363305984, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.42.attn_norm.weight", "offset": 10367238144, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 10367262720, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.42.ffn_gate.weight", "offset": 10430177280, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.42.ffn_up.weight", "offset": 10493091840, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.42.ffn_norm.weight", "offset": 10556006400, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 10556030976, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.42.attn_output.weight", "offset": 10559963136, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.42.attn_q.weight", "offset": 10583556096, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.42.attn_v.weight", "offset": 10607149056, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.43.ffn_gate.weight", "offset": 10611081216, "shape": [ 6144, 16384 ], "size": 3932160, "type": 3 }, { "name": "blk.43.ffn_up.weight", "offset": 10673995776, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.43.attn_k.weight", "offset": 10736910336, "shape": [ 6144, 1024 ], "size": 62914560, "type": 3 }, { "name": "blk.43.attn_output.weight", "offset": 10740842496, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.43.attn_q.weight", "offset": 10764435456, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.43.attn_v.weight", "offset": 10788028416, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.43.attn_norm.weight", "offset": 10791960576, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 10791985152, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.43.ffn_norm.weight", "offset": 10854899712, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 10854924288, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 10854948864, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.44.ffn_gate.weight", "offset": 10917863424, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.44.ffn_up.weight", "offset": 10980777984, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.44.ffn_norm.weight", "offset": 11043692544, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 11043717120, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.44.attn_output.weight", "offset": 11047649280, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.44.attn_q.weight", "offset": 11071242240, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.44.attn_v.weight", "offset": 11094835200, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.45.attn_norm.weight", "offset": 11098767360, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 11098791936, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.45.ffn_gate.weight", "offset": 11161706496, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.45.ffn_up.weight", "offset": 11224621056, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.45.ffn_norm.weight", "offset": 11287535616, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 11287560192, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.45.attn_output.weight", "offset": 11291492352, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.45.attn_q.weight", "offset": 11315085312, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.45.attn_v.weight", "offset": 11338678272, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.46.attn_norm.weight", "offset": 11342610432, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 11342635008, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.46.ffn_gate.weight", "offset": 11405549568, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.46.ffn_up.weight", "offset": 11468464128, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.46.ffn_norm.weight", "offset": 11531378688, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 11531403264, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.46.attn_output.weight", "offset": 11535335424, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.46.attn_q.weight", "offset": 11558928384, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.46.attn_v.weight", "offset": 11582521344, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.47.attn_norm.weight", "offset": 11586453504, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 11586478080, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.47.ffn_gate.weight", "offset": 11649392640, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.47.ffn_up.weight", "offset": 11712307200, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.47.ffn_norm.weight", "offset": 11775221760, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 11775246336, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.47.attn_output.weight", "offset": 11779178496, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.47.attn_q.weight", "offset": 11802771456, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.47.attn_v.weight", "offset": 11826364416, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.48.attn_norm.weight", "offset": 11830296576, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 11830321152, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.48.ffn_gate.weight", "offset": 11893235712, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "\ufffd\ufffdה\ufffd\r\u0018\ufffd\ufffd\ufffd9vܕq=)\u0006\ufffd8", "offset": 11956150272, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.48.ffn_norm.weight", "offset": 12019064832, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 12019089408, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.48.attn_output.weight", "offset": 12023021568, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.48.attn_q.weight", "offset": 12046614528, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.48.attn_v.weight", "offset": 12070207488, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.49.attn_norm.weight", "offset": 12074139648, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 12074164224, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.49.ffn_gate.weight", "offset": 12137078784, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.49.ffn_up.weight", "offset": 12199993344, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.49.ffn_norm.weight", "offset": 12262907904, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 12262932480, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.49.attn_output.weight", "offset": 12266864640, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.49.attn_q.weight", "offset": 12290457600, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.49.attn_v.weight", "offset": 12314050560, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.50.attn_k.weight", "offset": 12317982720, "shape": [ 6144, 1024 ], "size": 3932160, "type": 3 }, { "name": "blk.50.attn_q.weight", "offset": 12321914880, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.50.attn_v.weight", "offset": 12345507840, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "output.weight", "offset": 12349440000, "shape": [ 6144, 32768 ], "size": 3932160, "type": 14 }, { "name": "blk.50.attn_norm.weight", "offset": 12514590720, "shape": [ 6144 ], "size": 165150720, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 12514615296, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.50.ffn_gate.weight", "offset": 12577529856, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.50.ffn_up.weight", "offset": 12640444416, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.50.ffn_norm.weight", "offset": 12703358976, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 12703383552, "shape": [ 6144, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.51.attn_norm.weight", "offset": 12726976512, "shape": [ 6144 ], "size": 23592960, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 12727001088, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.51.ffn_gate.weight", "offset": 12789915648, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.51.ffn_up.weight", "offset": 12852830208, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.51.ffn_norm.weight", "offset": 12915744768, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 12915769344, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.51.attn_output.weight", "offset": 12919701504, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.51.attn_q.weight", "offset": 12943294464, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.51.attn_v.weight", "offset": 12966887424, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.52.attn_norm.weight", "offset": 12970819584, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 12970844160, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.52.ffn_gate.weight", "offset": 13033758720, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.52.ffn_up.weight", "offset": 13096673280, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.52.ffn_norm.weight", "offset": 13159587840, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 13159612416, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.52.attn_output.weight", "offset": 13163544576, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.52.attn_q.weight", "offset": 13187137536, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.52.attn_v.weight", "offset": 13210730496, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.53.attn_norm.weight", "offset": 13214662656, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 13214687232, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.53.ffn_gate.weight", "offset": 13277601792, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.53.ffn_up.weight", "offset": 13340516352, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.53.ffn_norm.weight", "offset": 13403430912, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 13403455488, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.53.attn_output.weight", "offset": 13407387648, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.53.attn_q.weight", "offset": 13430980608, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.53.attn_v.weight", "offset": 13454573568, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.54.attn_norm.weight", "offset": 13458505728, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 13458530304, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.54.ffn_gate.weight", "offset": 13521444864, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.54.ffn_up.weight", "offset": 13584359424, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.54.ffn_norm.weight", "offset": 13647273984, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 13647298560, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.54.attn_output.weight", "offset": 13651230720, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.54.attn_q.weight", "offset": 13674823680, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.54.attn_v.weight", "offset": 13698416640, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "blk.55.attn_norm.weight", "offset": 13702348800, "shape": [ 6144 ], "size": 3932160, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 13702373376, "shape": [ 16384, 6144 ], "size": 24576, "type": 3 }, { "name": "blk.55.ffn_gate.weight", "offset": 13765287936, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.55.ffn_up.weight", "offset": 13828202496, "shape": [ 6144, 16384 ], "size": 62914560, "type": 3 }, { "name": "blk.55.ffn_norm.weight", "offset": 13891117056, "shape": [ 6144 ], "size": 62914560, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 13891141632, "shape": [ 6144, 1024 ], "size": 24576, "type": 3 }, { "name": "blk.55.attn_output.weight", "offset": 13895073792, "shape": [ 6144, 6144 ], "size": 3932160, "type": 3 }, { "name": "blk.55.attn_q.weight", "offset": 13918666752, "shape": [ 6144, 6144 ], "size": 23592960, "type": 3 }, { "name": "blk.55.attn_v.weight", "offset": 13942259712, "shape": [ 6144, 1024 ], "size": 23592960, "type": 3 }, { "name": "output_norm.weight", "offset": 13946191872, "shape": [ 6144 ], "size": 3932160, "type": 0 } ], "version": 3 }