gemma2

{ "metadata": { "gemma2.attention.head_count": 32, "gemma2.attention.head_count_kv": 16, "gemma2.attention.key_length": 128, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 128, "gemma2.block_count": 46, "gemma2.context_length": 8192, "gemma2.embedding_length": 4608, "gemma2.feed_forward_length": 36864, "general.architecture": "gemma2", "general.file_type": 7, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 27227128320, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 18432, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.0.ffn_gate.weight", "offset": 180504576, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 360990720, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.0.post_attention_norm.weight", "offset": 541476864, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 541495296, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 541513728, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 541532160, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 551559168, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.0.attn_q.weight", "offset": 571613184, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.0.attn_v.weight", "offset": 591667200, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 601694208, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 601712640, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.1.ffn_gate.weight", "offset": 782198784, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.1.ffn_up.weight", "offset": 962684928, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.1.post_attention_norm.weight", "offset": 1143171072, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 1143189504, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 1143207936, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1143226368, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 1153253376, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.1.attn_q.weight", "offset": 1173307392, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.1.attn_v.weight", "offset": 1193361408, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.2.attn_k.weight", "offset": 1203388416, "shape": [ 4608, 2048 ], "size": 10027008, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 1213415424, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.2.attn_q.weight", "offset": 1233469440, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.2.attn_v.weight", "offset": 1253523456, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 1263550464, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1263568896, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.2.ffn_gate.weight", "offset": 1444055040, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.2.ffn_up.weight", "offset": 1624541184, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.2.post_attention_norm.weight", "offset": 1805027328, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 1805045760, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 1805064192, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_norm.weight", "offset": 1805082624, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1805101056, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.3.ffn_gate.weight", "offset": 1985587200, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.3.ffn_up.weight", "offset": 2166073344, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.3.post_attention_norm.weight", "offset": 2346559488, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 2346577920, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 2346596352, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 2346614784, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 2356641792, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.3.attn_q.weight", "offset": 2376695808, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.3.attn_v.weight", "offset": 2396749824, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 2406776832, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 2406795264, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.4.ffn_gate.weight", "offset": 2587281408, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.4.ffn_up.weight", "offset": 2767767552, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.4.post_attention_norm.weight", "offset": 2948253696, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 2948272128, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 2948290560, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 2948308992, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 2958336000, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.4.attn_q.weight", "offset": 2978390016, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.4.attn_v.weight", "offset": 2998444032, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.5.attn_norm.weight", "offset": 3008471040, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 3008489472, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.5.ffn_gate.weight", "offset": 3188975616, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.5.ffn_up.weight", "offset": 3369461760, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.5.post_attention_norm.weight", "offset": 3549947904, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 3549966336, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 3549984768, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 3550003200, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 3560030208, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.5.attn_q.weight", "offset": 3580084224, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.5.attn_v.weight", "offset": 3600138240, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.6.ffn_gate.weight", "offset": 3610165248, "shape": [ 4608, 36864 ], "size": 10027008, "type": 8 }, { "name": "blk.6.attn_k.weight", "offset": 3790651392, "shape": [ 4608, 2048 ], "size": 180486144, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 3800678400, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.6.attn_q.weight", "offset": 3820732416, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.6.attn_v.weight", "offset": 3840786432, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.6.attn_norm.weight", "offset": 3850813440, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 3850831872, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.6.ffn_up.weight", "offset": 4031318016, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.6.post_attention_norm.weight", "offset": 4211804160, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 4211822592, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 4211841024, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 4211859456, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 4211877888, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.7.ffn_gate.weight", "offset": 4392364032, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 4572850176, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.7.post_attention_norm.weight", "offset": 4753336320, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 4753354752, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 4753373184, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 4753391616, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 4763418624, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.7.attn_q.weight", "offset": 4783472640, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.7.attn_v.weight", "offset": 4803526656, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.8.attn_norm.weight", "offset": 4813553664, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 4813572096, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.8.ffn_gate.weight", "offset": 4994058240, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.8.ffn_up.weight", "offset": 5174544384, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.8.post_attention_norm.weight", "offset": 5355030528, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 5355048960, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 5355067392, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 5355085824, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 5365112832, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.8.attn_q.weight", "offset": 5385166848, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.8.attn_v.weight", "offset": 5405220864, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 5415247872, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 5415266304, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.9.ffn_gate.weight", "offset": 5595752448, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.9.ffn_up.weight", "offset": 5776238592, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.9.post_attention_norm.weight", "offset": 5956724736, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 5956743168, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 5956761600, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 5956780032, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 5966807040, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.9.attn_q.weight", "offset": 5986861056, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.9.attn_v.weight", "offset": 6006915072, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.10.ffn_gate.weight", "offset": 6016942080, "shape": [ 4608, 36864 ], "size": 10027008, "type": 8 }, { "name": "blk.10.ffn_up.weight", "offset": 6197428224, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.10.attn_k.weight", "offset": 6377914368, "shape": [ 4608, 2048 ], "size": 180486144, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 6387941376, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.10.attn_q.weight", "offset": 6407995392, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.10.attn_v.weight", "offset": 6428049408, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 6438076416, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 6438094848, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.10.post_attention_norm.weight", "offset": 6618580992, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 6618599424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 6618617856, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_norm.weight", "offset": 6618636288, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 6618654720, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.11.ffn_gate.weight", "offset": 6799140864, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.11.ffn_up.weight", "offset": 6979627008, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.11.post_attention_norm.weight", "offset": 7160113152, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 7160131584, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 7160150016, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 7160168448, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 7170195456, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.11.attn_q.weight", "offset": 7190249472, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.11.attn_v.weight", "offset": 7210303488, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.12.attn_norm.weight", "offset": 7220330496, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 7220348928, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.12.ffn_gate.weight", "offset": 7400835072, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.12.ffn_up.weight", "offset": 7581321216, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.12.post_attention_norm.weight", "offset": 7761807360, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 7761825792, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 7761844224, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 7761862656, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 7771889664, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.12.attn_q.weight", "offset": 7791943680, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.12.attn_v.weight", "offset": 7811997696, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.13.attn_norm.weight", "offset": 7822024704, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 7822043136, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.13.ffn_gate.weight", "offset": 8002529280, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.13.ffn_up.weight", "offset": 8183015424, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.13.post_attention_norm.weight", "offset": 8363501568, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 8363520000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 8363538432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 8363556864, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 8373583872, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.13.attn_q.weight", "offset": 8393637888, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.13.attn_v.weight", "offset": 8413691904, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 8423718912, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 8423737344, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.14.ffn_gate.weight", "offset": 8604223488, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.14.ffn_up.weight", "offset": 8784709632, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.14.post_attention_norm.weight", "offset": 8965195776, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 8965214208, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 8965232640, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 8965251072, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 8975278080, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.14.attn_q.weight", "offset": 8995332096, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.14.attn_v.weight", "offset": 9015386112, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.15.attn_k.weight", "offset": 9025413120, "shape": [ 4608, 2048 ], "size": 10027008, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 9035440128, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.15.attn_q.weight", "offset": 9055494144, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.15.attn_v.weight", "offset": 9075548160, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.15.attn_norm.weight", "offset": 9085575168, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 9085593600, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.15.ffn_gate.weight", "offset": 9266079744, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.15.ffn_up.weight", "offset": 9446565888, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.15.post_attention_norm.weight", "offset": 9627052032, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 9627070464, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 9627088896, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 9627107328, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 9627125760, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.16.ffn_gate.weight", "offset": 9807611904, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.16.ffn_up.weight", "offset": 9988098048, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.16.post_attention_norm.weight", "offset": 10168584192, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 10168602624, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 10168621056, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 10168639488, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 10178666496, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.16.attn_q.weight", "offset": 10198720512, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.16.attn_v.weight", "offset": 10218774528, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 10228801536, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 10228819968, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.17.ffn_gate.weight", "offset": 10409306112, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.17.ffn_up.weight", "offset": 10589792256, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.17.post_attention_norm.weight", "offset": 10770278400, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 10770296832, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 10770315264, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 10770333696, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 10780360704, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.17.attn_q.weight", "offset": 10800414720, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.17.attn_v.weight", "offset": 10820468736, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.18.attn_norm.weight", "offset": 10830495744, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 10830514176, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.18.ffn_gate.weight", "offset": 11011000320, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.18.ffn_up.weight", "offset": 11191486464, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.18.post_attention_norm.weight", "offset": 11371972608, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 11371991040, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 11372009472, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 11372027904, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 11382054912, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.18.attn_q.weight", "offset": 11402108928, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.18.attn_v.weight", "offset": 11422162944, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.19.ffn_gate.weight", "offset": 11432189952, "shape": [ 4608, 36864 ], "size": 10027008, "type": 8 }, { "name": "blk.19.attn_k.weight", "offset": 11612676096, "shape": [ 4608, 2048 ], "size": 180486144, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 11622703104, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.19.attn_q.weight", "offset": 11642757120, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.19.attn_v.weight", "offset": 11662811136, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.19.attn_norm.weight", "offset": 11672838144, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 11672856576, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.19.ffn_up.weight", "offset": 11853342720, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.19.post_attention_norm.weight", "offset": 12033828864, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 12033847296, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 12033865728, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_norm.weight", "offset": 12033884160, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 12033902592, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.20.ffn_gate.weight", "offset": 12214388736, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.20.ffn_up.weight", "offset": 12394874880, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.20.post_attention_norm.weight", "offset": 12575361024, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 12575379456, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 12575397888, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 12575416320, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 12585443328, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.20.attn_q.weight", "offset": 12605497344, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.20.attn_v.weight", "offset": 12625551360, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.21.attn_norm.weight", "offset": 12635578368, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 12635596800, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.21.ffn_gate.weight", "offset": 12816082944, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.21.ffn_up.weight", "offset": 12996569088, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.21.post_attention_norm.weight", "offset": 13177055232, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 13177073664, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 13177092096, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 13177110528, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 13187137536, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.21.attn_q.weight", "offset": 13207191552, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.21.attn_v.weight", "offset": 13227245568, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 13237272576, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 13237291008, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.22.ffn_gate.weight", "offset": 13417777152, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.22.ffn_up.weight", "offset": 13598263296, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.22.post_attention_norm.weight", "offset": 13778749440, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 13778767872, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 13778786304, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 13778804736, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 13788831744, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.22.attn_q.weight", "offset": 13808885760, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.22.attn_v.weight", "offset": 13828939776, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.23.ffn_up.weight", "offset": 13838966784, "shape": [ 4608, 36864 ], "size": 10027008, "type": 8 }, { "name": "blk.23.ffn_gate.weight", "offset": 14019452928, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.23.attn_k.weight", "offset": 14199939072, "shape": [ 4608, 2048 ], "size": 180486144, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 14209966080, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.23.attn_q.weight", "offset": 14230020096, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.23.attn_v.weight", "offset": 14250074112, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.23.attn_norm.weight", "offset": 14260101120, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 14260119552, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.23.post_attention_norm.weight", "offset": 14440605696, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 14440624128, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 14440642560, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 14440660992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 14440679424, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.24.ffn_gate.weight", "offset": 14621165568, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.24.ffn_up.weight", "offset": 14801651712, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.24.post_attention_norm.weight", "offset": 14982137856, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 14982156288, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 14982174720, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 14982193152, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 14992220160, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.24.attn_q.weight", "offset": 15012274176, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.24.attn_v.weight", "offset": 15032328192, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.25.attn_norm.weight", "offset": 15042355200, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 15042373632, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.25.ffn_gate.weight", "offset": 15222859776, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.25.ffn_up.weight", "offset": 15403345920, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.25.post_attention_norm.weight", "offset": 15583832064, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 15583850496, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 15583868928, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 15583887360, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 15593914368, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.25.attn_q.weight", "offset": 15613968384, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.25.attn_v.weight", "offset": 15634022400, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.26.attn_norm.weight", "offset": 15644049408, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 15644067840, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.26.ffn_gate.weight", "offset": 15824553984, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.26.ffn_up.weight", "offset": 16005040128, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.26.post_attention_norm.weight", "offset": 16185526272, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 16185544704, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 16185563136, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 16185581568, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 16195608576, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.26.attn_q.weight", "offset": 16215662592, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.26.attn_v.weight", "offset": 16235716608, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.27.attn_norm.weight", "offset": 16245743616, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 16245762048, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.27.ffn_gate.weight", "offset": 16426248192, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.27.ffn_up.weight", "offset": 16606734336, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.27.post_attention_norm.weight", "offset": 16787220480, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 16787238912, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 16787257344, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 16787275776, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 16797302784, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.27.attn_q.weight", "offset": 16817356800, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.27.attn_v.weight", "offset": 16837410816, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.28.attn_k.weight", "offset": 16847437824, "shape": [ 4608, 2048 ], "size": 10027008, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 16857464832, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.28.attn_q.weight", "offset": 16877518848, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.28.attn_v.weight", "offset": 16897572864, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.28.attn_norm.weight", "offset": 16907599872, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 16907618304, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.28.ffn_gate.weight", "offset": 17088104448, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.28.ffn_up.weight", "offset": 17268590592, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.28.post_attention_norm.weight", "offset": 17449076736, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 17449095168, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 17449113600, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 17449132032, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 17449150464, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.29.ffn_gate.weight", "offset": 17629636608, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.29.ffn_up.weight", "offset": 17810122752, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.29.post_attention_norm.weight", "offset": 17990608896, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 17990627328, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 17990645760, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 17990664192, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 18000691200, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.29.attn_q.weight", "offset": 18020745216, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.29.attn_v.weight", "offset": 18040799232, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.30.attn_norm.weight", "offset": 18050826240, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 18050844672, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.30.ffn_gate.weight", "offset": 18231330816, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.30.ffn_up.weight", "offset": 18411816960, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.30.post_attention_norm.weight", "offset": 18592303104, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 18592321536, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 18592339968, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 18592358400, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 18602385408, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.30.attn_q.weight", "offset": 18622439424, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.30.attn_v.weight", "offset": 18642493440, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.31.attn_norm.weight", "offset": 18652520448, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 18652538880, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.31.ffn_gate.weight", "offset": 18833025024, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.31.ffn_up.weight", "offset": 19013511168, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.31.post_attention_norm.weight", "offset": 19193997312, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 19194015744, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 19194034176, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 19194052608, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 19204079616, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.31.attn_q.weight", "offset": 19224133632, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.31.attn_v.weight", "offset": 19244187648, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.32.ffn_gate.weight", "offset": 19254214656, "shape": [ 4608, 36864 ], "size": 10027008, "type": 8 }, { "name": "blk.32.attn_k.weight", "offset": 19434700800, "shape": [ 4608, 2048 ], "size": 180486144, "type": 8 }, { "name": "blk.32.attn_output.weight", "offset": 19444727808, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.32.attn_q.weight", "offset": 19464781824, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.32.attn_v.weight", "offset": 19484835840, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.32.attn_norm.weight", "offset": 19494862848, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 19494881280, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.32.ffn_up.weight", "offset": 19675367424, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.32.post_attention_norm.weight", "offset": 19855853568, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 19855872000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 19855890432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 19855908864, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 19855927296, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.33.ffn_gate.weight", "offset": 20036413440, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.33.ffn_up.weight", "offset": 20216899584, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.33.post_attention_norm.weight", "offset": 20397385728, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 20397404160, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 20397422592, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 20397441024, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.33.attn_output.weight", "offset": 20407468032, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.33.attn_q.weight", "offset": 20427522048, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.33.attn_v.weight", "offset": 20447576064, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.34.ffn_norm.weight", "offset": 20457603072, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 20457621504, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.34.ffn_gate.weight", "offset": 20638107648, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.34.ffn_up.weight", "offset": 20818593792, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.34.post_attention_norm.weight", "offset": 20999079936, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 20999098368, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_norm.weight", "offset": 20999116800, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 20999135232, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.34.attn_output.weight", "offset": 21009162240, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.34.attn_q.weight", "offset": 21029216256, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.34.attn_v.weight", "offset": 21049270272, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.35.attn_norm.weight", "offset": 21059297280, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 21059315712, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.35.ffn_gate.weight", "offset": 21239801856, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.35.ffn_up.weight", "offset": 21420288000, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.35.post_attention_norm.weight", "offset": 21600774144, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 21600792576, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 21600811008, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 21600829440, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.35.attn_output.weight", "offset": 21610856448, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.35.attn_q.weight", "offset": 21630910464, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.35.attn_v.weight", "offset": 21650964480, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.36.ffn_gate.weight", "offset": 21660991488, "shape": [ 4608, 36864 ], "size": 10027008, "type": 8 }, { "name": "blk.36.ffn_up.weight", "offset": 21841477632, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.36.attn_k.weight", "offset": 22021963776, "shape": [ 4608, 2048 ], "size": 180486144, "type": 8 }, { "name": "blk.36.attn_output.weight", "offset": 22031990784, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.36.attn_q.weight", "offset": 22052044800, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.36.attn_v.weight", "offset": 22072098816, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.36.attn_norm.weight", "offset": 22082125824, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 22082144256, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.36.post_attention_norm.weight", "offset": 22262630400, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 22262648832, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 22262667264, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 22262685696, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 22262704128, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.37.ffn_gate.weight", "offset": 22443190272, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.37.ffn_up.weight", "offset": 22623676416, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.37.post_attention_norm.weight", "offset": 22804162560, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 22804180992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 22804199424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 22804217856, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.37.attn_output.weight", "offset": 22814244864, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.37.attn_q.weight", "offset": 22834298880, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.37.attn_v.weight", "offset": 22854352896, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.38.attn_norm.weight", "offset": 22864379904, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 22864398336, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.38.ffn_gate.weight", "offset": 23044884480, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.38.ffn_up.weight", "offset": 23225370624, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.38.post_attention_norm.weight", "offset": 23405856768, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 23405875200, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 23405893632, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 23405912064, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.38.attn_output.weight", "offset": 23415939072, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.38.attn_q.weight", "offset": 23435993088, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.38.attn_v.weight", "offset": 23456047104, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.39.attn_norm.weight", "offset": 23466074112, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 23466092544, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.39.ffn_gate.weight", "offset": 23646578688, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.39.ffn_up.weight", "offset": 23827064832, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.39.post_attention_norm.weight", "offset": 24007550976, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 24007569408, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 24007587840, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 24007606272, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.39.attn_output.weight", "offset": 24017633280, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.39.attn_q.weight", "offset": 24037687296, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.39.attn_v.weight", "offset": 24057741312, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.40.ffn_down.weight", "offset": 24067768320, "shape": [ 36864, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.40.attn_norm.weight", "offset": 24248254464, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.40.ffn_gate.weight", "offset": 24248272896, "shape": [ 4608, 36864 ], "size": 18432, "type": 8 }, { "name": "blk.40.ffn_up.weight", "offset": 24428759040, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.40.post_attention_norm.weight", "offset": 24609245184, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 24609263616, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 24609282048, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 24609300480, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.40.attn_output.weight", "offset": 24619327488, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.40.attn_q.weight", "offset": 24639381504, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.40.attn_v.weight", "offset": 24659435520, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.41.attn_k.weight", "offset": 24669462528, "shape": [ 4608, 2048 ], "size": 10027008, "type": 8 }, { "name": "blk.41.attn_output.weight", "offset": 24679489536, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.41.attn_q.weight", "offset": 24699543552, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.41.attn_v.weight", "offset": 24719597568, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.41.attn_norm.weight", "offset": 24729624576, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 24729643008, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.41.ffn_gate.weight", "offset": 24910129152, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.41.ffn_up.weight", "offset": 25090615296, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.41.post_attention_norm.weight", "offset": 25271101440, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 25271119872, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 25271138304, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 25271156736, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 25271175168, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.42.ffn_gate.weight", "offset": 25451661312, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.42.ffn_up.weight", "offset": 25632147456, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.42.post_attention_norm.weight", "offset": 25812633600, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.42.post_ffw_norm.weight", "offset": 25812652032, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 25812670464, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 25812688896, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.42.attn_output.weight", "offset": 25822715904, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.42.attn_q.weight", "offset": 25842769920, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.42.attn_v.weight", "offset": 25862823936, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.43.attn_norm.weight", "offset": 25872850944, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 25872869376, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.43.ffn_gate.weight", "offset": 26053355520, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.43.ffn_up.weight", "offset": 26233841664, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.43.post_attention_norm.weight", "offset": 26414327808, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.43.post_ffw_norm.weight", "offset": 26414346240, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 26414364672, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 26414383104, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.43.attn_output.weight", "offset": 26424410112, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.43.attn_q.weight", "offset": 26444464128, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.43.attn_v.weight", "offset": 26464518144, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.44.attn_norm.weight", "offset": 26474545152, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 26474563584, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.44.ffn_gate.weight", "offset": 26655049728, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.44.ffn_up.weight", "offset": 26835535872, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.44.post_attention_norm.weight", "offset": 27016022016, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.44.post_ffw_norm.weight", "offset": 27016040448, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 27016058880, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 27016077312, "shape": [ 4608, 2048 ], "size": 18432, "type": 8 }, { "name": "blk.44.attn_output.weight", "offset": 27026104320, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.44.attn_q.weight", "offset": 27046158336, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.44.attn_v.weight", "offset": 27066212352, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.45.ffn_gate.weight", "offset": 27076239360, "shape": [ 4608, 36864 ], "size": 10027008, "type": 8 }, { "name": "blk.45.attn_k.weight", "offset": 27256725504, "shape": [ 4608, 2048 ], "size": 180486144, "type": 8 }, { "name": "blk.45.attn_output.weight", "offset": 27266752512, "shape": [ 4096, 4608 ], "size": 10027008, "type": 8 }, { "name": "blk.45.attn_q.weight", "offset": 27286806528, "shape": [ 4608, 4096 ], "size": 20054016, "type": 8 }, { "name": "blk.45.attn_v.weight", "offset": 27306860544, "shape": [ 4608, 2048 ], "size": 20054016, "type": 8 }, { "name": "blk.45.attn_norm.weight", "offset": 27316887552, "shape": [ 4608 ], "size": 10027008, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 27316905984, "shape": [ 36864, 4608 ], "size": 18432, "type": 8 }, { "name": "blk.45.ffn_up.weight", "offset": 27497392128, "shape": [ 4608, 36864 ], "size": 180486144, "type": 8 }, { "name": "blk.45.post_attention_norm.weight", "offset": 27677878272, "shape": [ 4608 ], "size": 180486144, "type": 0 }, { "name": "blk.45.post_ffw_norm.weight", "offset": 27677896704, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 27677915136, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "output_norm.weight", "offset": 27677933568, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "token_embd.weight", "offset": 27677952000, "shape": [ 4608, 256000 ], "size": 18432, "type": 8 } ], "version": 3 }