codestral

{ "metadata": { "general.architecture": "llama", "general.file_type": 1, "general.name": "Codestral-22B-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 32768, "llama.embedding_length": 6144, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_space_prefix": true, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 22247282688, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 402653184, "shape": [ 6144 ], "size": 402653184, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 402677760, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.0.ffn_gate.weight", "offset": 604004352, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.0.ffn_up.weight", "offset": 805330944, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.0.ffn_norm.weight", "offset": 1006657536, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 1006682112, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.0.attn_output.weight", "offset": 1019265024, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.0.attn_q.weight", "offset": 1094762496, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.0.attn_v.weight", "offset": 1170259968, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.1.attn_norm.weight", "offset": 1182842880, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1182867456, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.1.ffn_gate.weight", "offset": 1384194048, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.1.ffn_up.weight", "offset": 1585520640, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.1.ffn_norm.weight", "offset": 1786847232, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1786871808, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.1.attn_output.weight", "offset": 1799454720, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.1.attn_q.weight", "offset": 1874952192, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.1.attn_v.weight", "offset": 1950449664, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 1963032576, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1963057152, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.2.ffn_gate.weight", "offset": 2164383744, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.2.ffn_up.weight", "offset": 2365710336, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.2.ffn_norm.weight", "offset": 2567036928, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 2567061504, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.2.attn_output.weight", "offset": 2579644416, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.2.attn_q.weight", "offset": 2655141888, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.2.attn_v.weight", "offset": 2730639360, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.3.attn_norm.weight", "offset": 2743222272, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 2743246848, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.3.ffn_gate.weight", "offset": 2944573440, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.3.ffn_up.weight", "offset": 3145900032, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.3.ffn_norm.weight", "offset": 3347226624, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 3347251200, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.3.attn_output.weight", "offset": 3359834112, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.3.attn_q.weight", "offset": 3435331584, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.3.attn_v.weight", "offset": 3510829056, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 3523411968, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 3523436544, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.4.ffn_gate.weight", "offset": 3724763136, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.4.ffn_up.weight", "offset": 3926089728, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.4.ffn_norm.weight", "offset": 4127416320, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 4127440896, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.4.attn_output.weight", "offset": 4140023808, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.4.attn_q.weight", "offset": 4215521280, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.4.attn_v.weight", "offset": 4291018752, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.5.ffn_gate.weight", "offset": 4303601664, "shape": [ 6144, 16384 ], "size": 12582912, "type": 1 }, { "name": "blk.5.ffn_up.weight", "offset": 4504928256, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.5.attn_k.weight", "offset": 4706254848, "shape": [ 6144, 1024 ], "size": 201326592, "type": 1 }, { "name": "blk.5.attn_output.weight", "offset": 4718837760, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.5.attn_q.weight", "offset": 4794335232, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.5.attn_v.weight", "offset": 4869832704, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.10.attn_norm.weight", "offset": 4882415616, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 4882440192, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.10.ffn_gate.weight", "offset": 5083766784, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.10.ffn_up.weight", "offset": 5285093376, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.10.ffn_norm.weight", "offset": 5486419968, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 5486444544, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.10.attn_output.weight", "offset": 5499027456, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.10.attn_q.weight", "offset": 5574524928, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.10.attn_v.weight", "offset": 5650022400, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.11.attn_norm.weight", "offset": 5662605312, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 5662629888, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.11.ffn_gate.weight", "offset": 5863956480, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.11.ffn_up.weight", "offset": 6065283072, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.11.ffn_norm.weight", "offset": 6266609664, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 6266634240, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.11.attn_output.weight", "offset": 6279217152, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.11.attn_q.weight", "offset": 6354714624, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.11.attn_v.weight", "offset": 6430212096, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.12.attn_k.weight", "offset": 6442795008, "shape": [ 6144, 1024 ], "size": 12582912, "type": 1 }, { "name": "blk.12.attn_q.weight", "offset": 6455377920, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.12.attn_v.weight", "offset": 6530875392, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.5.attn_norm.weight", "offset": 6543458304, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 6543482880, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.5.ffn_norm.weight", "offset": 6744809472, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 6744834048, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 6744858624, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.6.ffn_gate.weight", "offset": 6946185216, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.6.ffn_up.weight", "offset": 7147511808, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.6.ffn_norm.weight", "offset": 7348838400, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 7348862976, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.6.attn_output.weight", "offset": 7361445888, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.6.attn_q.weight", "offset": 7436943360, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.6.attn_v.weight", "offset": 7512440832, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.7.attn_norm.weight", "offset": 7525023744, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 7525048320, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.7.ffn_gate.weight", "offset": 7726374912, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.7.ffn_up.weight", "offset": 7927701504, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.7.ffn_norm.weight", "offset": 8129028096, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 8129052672, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.7.attn_output.weight", "offset": 8141635584, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.7.attn_q.weight", "offset": 8217133056, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.7.attn_v.weight", "offset": 8292630528, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.8.attn_norm.weight", "offset": 8305213440, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 8305238016, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.8.ffn_gate.weight", "offset": 8506564608, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.8.ffn_up.weight", "offset": 8707891200, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.8.ffn_norm.weight", "offset": 8909217792, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 8909242368, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.8.attn_output.weight", "offset": 8921825280, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.8.attn_q.weight", "offset": 8997322752, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.8.attn_v.weight", "offset": 9072820224, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 9085403136, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 9085427712, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.9.ffn_gate.weight", "offset": 9286754304, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.9.ffn_up.weight", "offset": 9488080896, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.9.ffn_norm.weight", "offset": 9689407488, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 9689432064, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.9.attn_output.weight", "offset": 9702014976, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.9.attn_q.weight", "offset": 9777512448, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.9.attn_v.weight", "offset": 9853009920, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 9865592832, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 9865617408, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.12.ffn_gate.weight", "offset": 10066944000, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.12.ffn_up.weight", "offset": 10268270592, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.12.ffn_norm.weight", "offset": 10469597184, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 10469621760, "shape": [ 6144, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.13.attn_norm.weight", "offset": 10545119232, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 10545143808, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.13.ffn_gate.weight", "offset": 10746470400, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.13.ffn_up.weight", "offset": 10947796992, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.13.ffn_norm.weight", "offset": 11149123584, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 11149148160, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.13.attn_output.weight", "offset": 11161731072, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.13.attn_q.weight", "offset": 11237228544, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.13.attn_v.weight", "offset": 11312726016, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 11325308928, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 11325333504, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.14.ffn_gate.weight", "offset": 11526660096, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.14.ffn_up.weight", "offset": 11727986688, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.14.ffn_norm.weight", "offset": 11929313280, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 11929337856, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.14.attn_output.weight", "offset": 11941920768, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.14.attn_q.weight", "offset": 12017418240, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.14.attn_v.weight", "offset": 12092915712, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.15.attn_norm.weight", "offset": 12105498624, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 12105523200, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.15.ffn_gate.weight", "offset": 12306849792, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.15.ffn_up.weight", "offset": 12508176384, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.15.ffn_norm.weight", "offset": 12709502976, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 12709527552, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.15.attn_output.weight", "offset": 12722110464, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.15.attn_q.weight", "offset": 12797607936, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.15.attn_v.weight", "offset": 12873105408, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.16.attn_norm.weight", "offset": 12885688320, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 12885712896, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.16.ffn_gate.weight", "offset": 13087039488, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.16.ffn_up.weight", "offset": 13288366080, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.16.ffn_norm.weight", "offset": 13489692672, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 13489717248, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.16.attn_output.weight", "offset": 13502300160, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.16.attn_q.weight", "offset": 13577797632, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.16.attn_v.weight", "offset": 13653295104, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 13665878016, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 13665902592, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.17.ffn_gate.weight", "offset": 13867229184, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.17.ffn_up.weight", "offset": 14068555776, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.17.ffn_norm.weight", "offset": 14269882368, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 14269906944, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.17.attn_output.weight", "offset": 14282489856, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.17.attn_q.weight", "offset": 14357987328, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.17.attn_v.weight", "offset": 14433484800, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.18.ffn_gate.weight", "offset": 14446067712, "shape": [ 6144, 16384 ], "size": 12582912, "type": 1 }, { "name": "blk.18.attn_k.weight", "offset": 14647394304, "shape": [ 6144, 1024 ], "size": 201326592, "type": 1 }, { "name": "blk.18.attn_output.weight", "offset": 14659977216, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.18.attn_q.weight", "offset": 14735474688, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.18.attn_v.weight", "offset": 14810972160, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.18.attn_norm.weight", "offset": 14823555072, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 14823579648, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.18.ffn_up.weight", "offset": 15024906240, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.18.ffn_norm.weight", "offset": 15226232832, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 15226257408, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 15226281984, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.19.ffn_gate.weight", "offset": 15427608576, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.19.ffn_up.weight", "offset": 15628935168, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.19.ffn_norm.weight", "offset": 15830261760, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 15830286336, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.19.attn_output.weight", "offset": 15842869248, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.19.attn_q.weight", "offset": 15918366720, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.19.attn_v.weight", "offset": 15993864192, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.20.attn_norm.weight", "offset": 16006447104, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 16006471680, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.20.ffn_gate.weight", "offset": 16207798272, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.20.ffn_up.weight", "offset": 16409124864, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.20.ffn_norm.weight", "offset": 16610451456, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 16610476032, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.20.attn_output.weight", "offset": 16623058944, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.20.attn_q.weight", "offset": 16698556416, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.20.attn_v.weight", "offset": 16774053888, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 16786636800, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 16786661376, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.21.ffn_gate.weight", "offset": 16987987968, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.21.ffn_up.weight", "offset": 17189314560, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.21.ffn_norm.weight", "offset": 17390641152, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 17390665728, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.21.attn_output.weight", "offset": 17403248640, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.21.attn_q.weight", "offset": 17478746112, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.21.attn_v.weight", "offset": 17554243584, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 17566826496, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 17566851072, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.22.ffn_gate.weight", "offset": 17768177664, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.22.ffn_up.weight", "offset": 17969504256, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.22.ffn_norm.weight", "offset": 18170830848, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 18170855424, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.22.attn_output.weight", "offset": 18183438336, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.22.attn_q.weight", "offset": 18258935808, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.22.attn_v.weight", "offset": 18334433280, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 18347016192, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 18347040768, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.23.ffn_gate.weight", "offset": 18548367360, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.23.ffn_up.weight", "offset": 18749693952, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.23.ffn_norm.weight", "offset": 18951020544, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 18951045120, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.23.attn_output.weight", "offset": 18963628032, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.23.attn_q.weight", "offset": 19039125504, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.23.attn_v.weight", "offset": 19114622976, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.24.ffn_gate.weight", "offset": 19127205888, "shape": [ 6144, 16384 ], "size": 12582912, "type": 1 }, { "name": "blk.24.ffn_up.weight", "offset": 19328532480, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.24.attn_k.weight", "offset": 19529859072, "shape": [ 6144, 1024 ], "size": 201326592, "type": 1 }, { "name": "blk.24.attn_output.weight", "offset": 19542441984, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.24.attn_q.weight", "offset": 19617939456, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.24.attn_v.weight", "offset": 19693436928, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 19706019840, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 19706044416, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.24.ffn_norm.weight", "offset": 19907371008, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.25.attn_norm.weight", "offset": 19907395584, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 19907420160, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.25.ffn_gate.weight", "offset": 20108746752, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.25.ffn_up.weight", "offset": 20310073344, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.25.ffn_norm.weight", "offset": 20511399936, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 20511424512, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.25.attn_output.weight", "offset": 20524007424, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.25.attn_q.weight", "offset": 20599504896, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.25.attn_v.weight", "offset": 20675002368, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 20687585280, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 20687609856, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.26.ffn_gate.weight", "offset": 20888936448, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.26.ffn_up.weight", "offset": 21090263040, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.26.ffn_norm.weight", "offset": 21291589632, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 21291614208, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.26.attn_output.weight", "offset": 21304197120, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.26.attn_q.weight", "offset": 21379694592, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.26.attn_v.weight", "offset": 21455192064, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.27.attn_norm.weight", "offset": 21467774976, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 21467799552, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.27.ffn_gate.weight", "offset": 21669126144, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.27.ffn_up.weight", "offset": 21870452736, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.27.ffn_norm.weight", "offset": 22071779328, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 22071803904, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.27.attn_output.weight", "offset": 22084386816, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.27.attn_q.weight", "offset": 22159884288, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.27.attn_v.weight", "offset": 22235381760, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.28.attn_norm.weight", "offset": 22247964672, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 22247989248, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.28.ffn_gate.weight", "offset": 22449315840, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.28.ffn_up.weight", "offset": 22650642432, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.28.ffn_norm.weight", "offset": 22851969024, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 22851993600, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.28.attn_output.weight", "offset": 22864576512, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.28.attn_q.weight", "offset": 22940073984, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.28.attn_v.weight", "offset": 23015571456, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.29.attn_norm.weight", "offset": 23028154368, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 23028178944, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.29.ffn_gate.weight", "offset": 23229505536, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.29.ffn_up.weight", "offset": 23430832128, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.29.ffn_norm.weight", "offset": 23632158720, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 23632183296, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.29.attn_output.weight", "offset": 23644766208, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.29.attn_q.weight", "offset": 23720263680, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.29.attn_v.weight", "offset": 23795761152, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.30.attn_norm.weight", "offset": 23808344064, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 23808368640, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.30.ffn_gate.weight", "offset": 24009695232, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.30.ffn_up.weight", "offset": 24211021824, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.30.ffn_norm.weight", "offset": 24412348416, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 24412372992, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.30.attn_output.weight", "offset": 24424955904, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.30.attn_q.weight", "offset": 24500453376, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.30.attn_v.weight", "offset": 24575950848, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.31.attn_k.weight", "offset": 24588533760, "shape": [ 6144, 1024 ], "size": 12582912, "type": 1 }, { "name": "blk.31.attn_q.weight", "offset": 24601116672, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.31.attn_v.weight", "offset": 24676614144, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.31.attn_norm.weight", "offset": 24689197056, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 24689221632, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.31.ffn_gate.weight", "offset": 24890548224, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.31.ffn_up.weight", "offset": 25091874816, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.31.ffn_norm.weight", "offset": 25293201408, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 25293225984, "shape": [ 6144, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.32.attn_norm.weight", "offset": 25368723456, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 25368748032, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.32.ffn_gate.weight", "offset": 25570074624, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.32.ffn_up.weight", "offset": 25771401216, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.32.ffn_norm.weight", "offset": 25972727808, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 25972752384, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.32.attn_output.weight", "offset": 25985335296, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.32.attn_q.weight", "offset": 26060832768, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.32.attn_v.weight", "offset": 26136330240, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.33.attn_norm.weight", "offset": 26148913152, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 26148937728, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.33.ffn_gate.weight", "offset": 26350264320, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.33.ffn_up.weight", "offset": 26551590912, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.33.ffn_norm.weight", "offset": 26752917504, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 26752942080, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.33.attn_output.weight", "offset": 26765524992, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.33.attn_q.weight", "offset": 26841022464, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.33.attn_v.weight", "offset": 26916519936, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.34.attn_norm.weight", "offset": 26929102848, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 26929127424, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.34.ffn_gate.weight", "offset": 27130454016, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.34.ffn_up.weight", "offset": 27331780608, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.34.ffn_norm.weight", "offset": 27533107200, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 27533131776, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.34.attn_output.weight", "offset": 27545714688, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.34.attn_q.weight", "offset": 27621212160, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.34.attn_v.weight", "offset": 27696709632, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.35.attn_norm.weight", "offset": 27709292544, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 27709317120, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.35.ffn_gate.weight", "offset": 27910643712, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.35.ffn_up.weight", "offset": 28111970304, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.35.ffn_norm.weight", "offset": 28313296896, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 28313321472, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.35.attn_output.weight", "offset": 28325904384, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.35.attn_q.weight", "offset": 28401401856, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.35.attn_v.weight", "offset": 28476899328, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.36.attn_norm.weight", "offset": 28489482240, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 28489506816, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.36.ffn_gate.weight", "offset": 28690833408, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.36.ffn_up.weight", "offset": 28892160000, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.36.ffn_norm.weight", "offset": 29093486592, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 29093511168, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.36.attn_output.weight", "offset": 29106094080, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.36.attn_q.weight", "offset": 29181591552, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.36.attn_v.weight", "offset": 29257089024, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.37.ffn_gate.weight", "offset": 29269671936, "shape": [ 6144, 16384 ], "size": 12582912, "type": 1 }, { "name": "blk.37.attn_k.weight", "offset": 29470998528, "shape": [ 6144, 1024 ], "size": 201326592, "type": 1 }, { "name": "blk.37.attn_output.weight", "offset": 29483581440, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.37.attn_q.weight", "offset": 29559078912, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.37.attn_v.weight", "offset": 29634576384, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.37.attn_norm.weight", "offset": 29647159296, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 29647183872, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.37.ffn_up.weight", "offset": 29848510464, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.37.ffn_norm.weight", "offset": 30049837056, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.38.attn_norm.weight", "offset": 30049861632, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 30049886208, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.38.ffn_gate.weight", "offset": 30251212800, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.38.ffn_up.weight", "offset": 30452539392, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.38.ffn_norm.weight", "offset": 30653865984, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 30653890560, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.38.attn_output.weight", "offset": 30666473472, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.38.attn_q.weight", "offset": 30741970944, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.38.attn_v.weight", "offset": 30817468416, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.39.attn_norm.weight", "offset": 30830051328, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 30830075904, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.39.ffn_gate.weight", "offset": 31031402496, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.39.ffn_up.weight", "offset": 31232729088, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.39.ffn_norm.weight", "offset": 31434055680, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 31434080256, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.39.attn_output.weight", "offset": 31446663168, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.39.attn_q.weight", "offset": 31522160640, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.39.attn_v.weight", "offset": 31597658112, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.40.attn_norm.weight", "offset": 31610241024, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 31610265600, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.40.ffn_gate.weight", "offset": 31811592192, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.40.ffn_up.weight", "offset": 32012918784, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.40.ffn_norm.weight", "offset": 32214245376, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 32214269952, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.40.attn_output.weight", "offset": 32226852864, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.40.attn_q.weight", "offset": 32302350336, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.40.attn_v.weight", "offset": 32377847808, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.41.attn_norm.weight", "offset": 32390430720, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 32390455296, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.41.ffn_gate.weight", "offset": 32591781888, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.41.ffn_up.weight", "offset": 32793108480, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.41.ffn_norm.weight", "offset": 32994435072, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 32994459648, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.41.attn_output.weight", "offset": 33007042560, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.41.attn_q.weight", "offset": 33082540032, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.41.attn_v.weight", "offset": 33158037504, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.42.attn_norm.weight", "offset": 33170620416, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 33170644992, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.42.ffn_gate.weight", "offset": 33371971584, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.42.ffn_up.weight", "offset": 33573298176, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.42.ffn_norm.weight", "offset": 33774624768, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 33774649344, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.42.attn_output.weight", "offset": 33787232256, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.42.attn_q.weight", "offset": 33862729728, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.42.attn_v.weight", "offset": 33938227200, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.43.ffn_gate.weight", "offset": 33950810112, "shape": [ 6144, 16384 ], "size": 12582912, "type": 1 }, { "name": "blk.43.ffn_up.weight", "offset": 34152136704, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.43.attn_k.weight", "offset": 34353463296, "shape": [ 6144, 1024 ], "size": 201326592, "type": 1 }, { "name": "blk.43.attn_output.weight", "offset": 34366046208, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.43.attn_q.weight", "offset": 34441543680, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.43.attn_v.weight", "offset": 34517041152, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.43.attn_norm.weight", "offset": 34529624064, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 34529648640, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.43.ffn_norm.weight", "offset": 34730975232, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 34730999808, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 34731024384, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.44.ffn_gate.weight", "offset": 34932350976, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.44.ffn_up.weight", "offset": 35133677568, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.44.ffn_norm.weight", "offset": 35335004160, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 35335028736, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.44.attn_output.weight", "offset": 35347611648, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.44.attn_q.weight", "offset": 35423109120, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.44.attn_v.weight", "offset": 35498606592, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.45.attn_norm.weight", "offset": 35511189504, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 35511214080, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.45.ffn_gate.weight", "offset": 35712540672, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.45.ffn_up.weight", "offset": 35913867264, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.45.ffn_norm.weight", "offset": 36115193856, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 36115218432, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.45.attn_output.weight", "offset": 36127801344, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.45.attn_q.weight", "offset": 36203298816, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.45.attn_v.weight", "offset": 36278796288, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.46.attn_norm.weight", "offset": 36291379200, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 36291403776, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.46.ffn_gate.weight", "offset": 36492730368, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.46.ffn_up.weight", "offset": 36694056960, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.46.ffn_norm.weight", "offset": 36895383552, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 36895408128, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.46.attn_output.weight", "offset": 36907991040, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.46.attn_q.weight", "offset": 36983488512, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.46.attn_v.weight", "offset": 37058985984, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.47.attn_norm.weight", "offset": 37071568896, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 37071593472, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.47.ffn_gate.weight", "offset": 37272920064, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.47.ffn_up.weight", "offset": 37474246656, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.47.ffn_norm.weight", "offset": 37675573248, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 37675597824, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.47.attn_output.weight", "offset": 37688180736, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.47.attn_q.weight", "offset": 37763678208, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.47.attn_v.weight", "offset": 37839175680, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.48.attn_norm.weight", "offset": 37851758592, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 37851783168, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.48.ffn_gate.weight", "offset": 38053109760, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.48.ffn_up.weight", "offset": 38254436352, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.48.ffn_norm.weight", "offset": 38455762944, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 38455787520, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.48.attn_output.weight", "offset": 38468370432, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.48.attn_q.weight", "offset": 38543867904, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.48.attn_v.weight", "offset": 38619365376, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.49.attn_norm.weight", "offset": 38631948288, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 38631972864, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.49.ffn_gate.weight", "offset": 38833299456, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.49.ffn_up.weight", "offset": 39034626048, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.49.ffn_norm.weight", "offset": 39235952640, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 39235977216, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.49.attn_output.weight", "offset": 39248560128, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.49.attn_q.weight", "offset": 39324057600, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.49.attn_v.weight", "offset": 39399555072, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.50.attn_k.weight", "offset": 39412137984, "shape": [ 6144, 1024 ], "size": 12582912, "type": 1 }, { "name": "blk.50.attn_q.weight", "offset": 39424720896, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.50.attn_v.weight", "offset": 39500218368, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "output.weight", "offset": 39512801280, "shape": [ 6144, 32768 ], "size": 12582912, "type": 1 }, { "name": "blk.50.attn_norm.weight", "offset": 39915454464, "shape": [ 6144 ], "size": 402653184, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 39915479040, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.50.ffn_gate.weight", "offset": 40116805632, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.50.ffn_up.weight", "offset": 40318132224, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.50.ffn_norm.weight", "offset": 40519458816, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 40519483392, "shape": [ 6144, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.51.attn_norm.weight", "offset": 40594980864, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 40595005440, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.51.ffn_gate.weight", "offset": 40796332032, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.51.ffn_up.weight", "offset": 40997658624, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.51.ffn_norm.weight", "offset": 41198985216, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 41199009792, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.51.attn_output.weight", "offset": 41211592704, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.51.attn_q.weight", "offset": 41287090176, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.51.attn_v.weight", "offset": 41362587648, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.52.attn_norm.weight", "offset": 41375170560, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 41375195136, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.52.ffn_gate.weight", "offset": 41576521728, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.52.ffn_up.weight", "offset": 41777848320, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.52.ffn_norm.weight", "offset": 41979174912, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 41979199488, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.52.attn_output.weight", "offset": 41991782400, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.52.attn_q.weight", "offset": 42067279872, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.52.attn_v.weight", "offset": 42142777344, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.53.attn_norm.weight", "offset": 42155360256, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 42155384832, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.53.ffn_gate.weight", "offset": 42356711424, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.53.ffn_up.weight", "offset": 42558038016, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.53.ffn_norm.weight", "offset": 42759364608, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 42759389184, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.53.attn_output.weight", "offset": 42771972096, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.53.attn_q.weight", "offset": 42847469568, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.53.attn_v.weight", "offset": 42922967040, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.54.attn_norm.weight", "offset": 42935549952, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 42935574528, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.54.ffn_gate.weight", "offset": 43136901120, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.54.ffn_up.weight", "offset": 43338227712, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.54.ffn_norm.weight", "offset": 43539554304, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 43539578880, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.54.attn_output.weight", "offset": 43552161792, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.54.attn_q.weight", "offset": 43627659264, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.54.attn_v.weight", "offset": 43703156736, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "blk.55.attn_norm.weight", "offset": 43715739648, "shape": [ 6144 ], "size": 12582912, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 43715764224, "shape": [ 16384, 6144 ], "size": 24576, "type": 1 }, { "name": "blk.55.ffn_gate.weight", "offset": 43917090816, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.55.ffn_up.weight", "offset": 44118417408, "shape": [ 6144, 16384 ], "size": 201326592, "type": 1 }, { "name": "blk.55.ffn_norm.weight", "offset": 44319744000, "shape": [ 6144 ], "size": 201326592, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 44319768576, "shape": [ 6144, 1024 ], "size": 24576, "type": 1 }, { "name": "blk.55.attn_output.weight", "offset": 44332351488, "shape": [ 6144, 6144 ], "size": 12582912, "type": 1 }, { "name": "blk.55.attn_q.weight", "offset": 44407848960, "shape": [ 6144, 6144 ], "size": 75497472, "type": 1 }, { "name": "blk.55.attn_v.weight", "offset": 44483346432, "shape": [ 6144, 1024 ], "size": 75497472, "type": 1 }, { "name": "output_norm.weight", "offset": 44495929344, "shape": [ 6144 ], "size": 12582912, "type": 0 } ], "version": 3 }