gemma2

{ "metadata": { "gemma2.attention.head_count": 32, "gemma2.attention.head_count_kv": 16, "gemma2.attention.key_length": 128, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 128, "gemma2.block_count": 46, "gemma2.context_length": 8192, "gemma2.embedding_length": 4608, "gemma2.feed_forward_length": 36864, "general.architecture": "gemma2", "general.file_type": 17, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 27227128320, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 18432, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 139364352, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 256149504, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.0.post_attention_norm.weight", "offset": 372934656, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 372953088, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 372971520, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 372989952, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 379478016, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 392454144, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.0.attn_v.weight", "offset": 405430272, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 413171712, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 413190144, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 552536064, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 669321216, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.1.post_attention_norm.weight", "offset": 786106368, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 786124800, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 786143232, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 786161664, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 792649728, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 805625856, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.1.attn_v.weight", "offset": 818601984, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.2.attn_k.weight", "offset": 826343424, "shape": [ 4608, 2048 ], "size": 7741440, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 832831488, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 845807616, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.2.attn_v.weight", "offset": 858783744, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 866525184, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 866543616, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 1005889536, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 1122674688, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.2.post_attention_norm.weight", "offset": 1239459840, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 1239478272, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 1239496704, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_norm.weight", "offset": 1239515136, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1239533568, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 1378879488, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 1495664640, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.3.post_attention_norm.weight", "offset": 1612449792, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 1612468224, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 1612486656, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1612505088, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 1618993152, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 1631969280, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.3.attn_v.weight", "offset": 1644945408, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1652686848, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1652705280, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 1792051200, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 1908836352, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.4.post_attention_norm.weight", "offset": 2025621504, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 2025639936, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 2025658368, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 2025676800, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 2032164864, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 2045140992, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.4.attn_v.weight", "offset": 2058117120, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 2065858560, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 2065876992, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 2182662144, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 2299447296, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.5.post_attention_norm.weight", "offset": 2416232448, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 2416250880, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 2416269312, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 2416287744, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 2422775808, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 2435751936, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.5.attn_v.weight", "offset": 2448728064, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 2455216128, "shape": [ 4608, 36864 ], "size": 6488064, "type": 13 }, { "name": "blk.6.attn_k.weight", "offset": 2572001280, "shape": [ 4608, 2048 ], "size": 116785152, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 2578489344, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 2591465472, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.6.attn_v.weight", "offset": 2604441600, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 2610929664, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2610948096, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 2727733248, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.6.post_attention_norm.weight", "offset": 2844518400, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 2844536832, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 2844555264, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2844573696, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2844592128, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 2983938048, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.7.ffn_up.weight", "offset": 3100723200, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.7.post_attention_norm.weight", "offset": 3217508352, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 3217526784, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 3217545216, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 3217563648, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 3224051712, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 3237027840, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.7.attn_v.weight", "offset": 3250003968, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.8.attn_norm.weight", "offset": 3257745408, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3257763840, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 3374548992, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 3491334144, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.8.post_attention_norm.weight", "offset": 3608119296, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 3608137728, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 3608156160, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 3608174592, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 3614662656, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 3627638784, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.8.attn_v.weight", "offset": 3640614912, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 3647102976, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3647121408, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 3763906560, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.9.ffn_up.weight", "offset": 3880691712, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.9.post_attention_norm.weight", "offset": 3997476864, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 3997495296, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 3997513728, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 3997532160, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 4004020224, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 4016996352, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.9.attn_v.weight", "offset": 4029972480, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 4036460544, "shape": [ 4608, 36864 ], "size": 6488064, "type": 13 }, { "name": "blk.10.ffn_up.weight", "offset": 4153245696, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.10.attn_k.weight", "offset": 4270030848, "shape": [ 4608, 2048 ], "size": 116785152, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 4276518912, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 4289495040, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.10.attn_v.weight", "offset": 4302471168, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 4310212608, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 4310231040, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.10.post_attention_norm.weight", "offset": 4449576960, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 4449595392, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 4449613824, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_norm.weight", "offset": 4449632256, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 4449650688, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 4566435840, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.11.ffn_up.weight", "offset": 4683220992, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.11.post_attention_norm.weight", "offset": 4800006144, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 4800024576, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 4800043008, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 4800061440, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 4806549504, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 4819525632, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.11.attn_v.weight", "offset": 4832501760, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 4838989824, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 4839008256, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 4955793408, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.12.ffn_up.weight", "offset": 5072578560, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.12.post_attention_norm.weight", "offset": 5189363712, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 5189382144, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 5189400576, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 5189419008, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 5195907072, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 5208883200, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.12.attn_v.weight", "offset": 5221859328, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 5228347392, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 5228365824, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.13.ffn_gate.weight", "offset": 5367711744, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.13.ffn_up.weight", "offset": 5484496896, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.13.post_attention_norm.weight", "offset": 5601282048, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 5601300480, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 5601318912, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 5601337344, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 5607825408, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 5620801536, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.13.attn_v.weight", "offset": 5633777664, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 5641519104, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 5641537536, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 5758322688, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 5875107840, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.14.post_attention_norm.weight", "offset": 5991892992, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 5991911424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 5991929856, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 5991948288, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 5998436352, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 6011412480, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.14.attn_v.weight", "offset": 6024388608, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.15.attn_k.weight", "offset": 6030876672, "shape": [ 4608, 2048 ], "size": 6488064, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 6037364736, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 6050340864, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.15.attn_v.weight", "offset": 6063316992, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 6069805056, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 6069823488, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 6186608640, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 6303393792, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.15.post_attention_norm.weight", "offset": 6420178944, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 6420197376, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 6420215808, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 6420234240, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 6420252672, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 6559598592, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.16.ffn_up.weight", "offset": 6676383744, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.16.post_attention_norm.weight", "offset": 6793168896, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 6793187328, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 6793205760, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 6793224192, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 6799712256, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 6812688384, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.16.attn_v.weight", "offset": 6825664512, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 6833405952, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 6833424384, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 6950209536, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.17.ffn_up.weight", "offset": 7066994688, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.17.post_attention_norm.weight", "offset": 7183779840, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 7183798272, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 7183816704, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 7183835136, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 7190323200, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 7203299328, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.17.attn_v.weight", "offset": 7216275456, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 7222763520, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 7222781952, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 7339567104, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 7456352256, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.18.post_attention_norm.weight", "offset": 7573137408, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 7573155840, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 7573174272, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 7573192704, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 7579680768, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 7592656896, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.18.attn_v.weight", "offset": 7605633024, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 7612121088, "shape": [ 4608, 36864 ], "size": 6488064, "type": 13 }, { "name": "blk.19.attn_k.weight", "offset": 7728906240, "shape": [ 4608, 2048 ], "size": 116785152, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 7735394304, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 7748370432, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.19.attn_v.weight", "offset": 7761346560, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 7769088000, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 7769106432, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 7908452352, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.19.post_attention_norm.weight", "offset": 8025237504, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 8025255936, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 8025274368, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_norm.weight", "offset": 8025292800, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 8025311232, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 8142096384, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 8258881536, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.20.post_attention_norm.weight", "offset": 8375666688, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 8375685120, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 8375703552, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 8375721984, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 8382210048, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 8395186176, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.20.attn_v.weight", "offset": 8408162304, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 8414650368, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 8414668800, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.21.ffn_gate.weight", "offset": 8531453952, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 8648239104, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.21.post_attention_norm.weight", "offset": 8765024256, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 8765042688, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 8765061120, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 8765079552, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 8771567616, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 8784543744, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.21.attn_v.weight", "offset": 8797519872, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 8804007936, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 8804026368, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.22.ffn_gate.weight", "offset": 8943372288, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.22.ffn_up.weight", "offset": 9060157440, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.22.post_attention_norm.weight", "offset": 9176942592, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 9176961024, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 9176979456, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 9176997888, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 9183485952, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 9196462080, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.22.attn_v.weight", "offset": 9209438208, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.23.ffn_up.weight", "offset": 9217179648, "shape": [ 4608, 36864 ], "size": 7741440, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 9333964800, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.23.attn_k.weight", "offset": 9450749952, "shape": [ 4608, 2048 ], "size": 116785152, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 9457238016, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.23.attn_q.weight", "offset": 9470214144, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.23.attn_v.weight", "offset": 9483190272, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 9489678336, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 9489696768, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.23.post_attention_norm.weight", "offset": 9606481920, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 9606500352, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 9606518784, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 9606537216, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 9606555648, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.24.ffn_gate.weight", "offset": 9723340800, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 9840125952, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.24.post_attention_norm.weight", "offset": 9956911104, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 9956929536, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 9956947968, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 9956966400, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 9963454464, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.24.attn_q.weight", "offset": 9976430592, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.24.attn_v.weight", "offset": 9989406720, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 9995894784, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 9995913216, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 10135259136, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.25.ffn_up.weight", "offset": 10252044288, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.25.post_attention_norm.weight", "offset": 10368829440, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 10368847872, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 10368866304, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 10368884736, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 10375372800, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 10388348928, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.25.attn_v.weight", "offset": 10401325056, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 10409066496, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 10409084928, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 10525870080, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 10642655232, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.26.post_attention_norm.weight", "offset": 10759440384, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 10759458816, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 10759477248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 10759495680, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 10765983744, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 10778959872, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.26.attn_v.weight", "offset": 10791936000, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 10798424064, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 10798442496, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 10915227648, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 11032012800, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.27.post_attention_norm.weight", "offset": 11148797952, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 11148816384, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 11148834816, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 11148853248, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.27.attn_output.weight", "offset": 11155341312, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.27.attn_q.weight", "offset": 11168317440, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.27.attn_v.weight", "offset": 11181293568, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.28.attn_k.weight", "offset": 11187781632, "shape": [ 4608, 2048 ], "size": 6488064, "type": 13 }, { "name": "blk.28.attn_output.weight", "offset": 11194269696, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 11207245824, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.28.attn_v.weight", "offset": 11220221952, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.28.attn_norm.weight", "offset": 11227963392, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 11227981824, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.28.ffn_gate.weight", "offset": 11367327744, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.28.ffn_up.weight", "offset": 11484112896, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.28.post_attention_norm.weight", "offset": 11600898048, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 11600916480, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 11600934912, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 11600953344, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 11600971776, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.29.ffn_gate.weight", "offset": 11717756928, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.29.ffn_up.weight", "offset": 11834542080, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.29.post_attention_norm.weight", "offset": 11951327232, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 11951345664, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 11951364096, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 11951382528, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.29.attn_output.weight", "offset": 11957870592, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 11970846720, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.29.attn_v.weight", "offset": 11983822848, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 11990310912, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 11990329344, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.30.ffn_gate.weight", "offset": 12107114496, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.30.ffn_up.weight", "offset": 12223899648, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.30.post_attention_norm.weight", "offset": 12340684800, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 12340703232, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 12340721664, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 12340740096, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.30.attn_output.weight", "offset": 12347228160, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 12360204288, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.30.attn_v.weight", "offset": 12373180416, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 12379668480, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 12379686912, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.31.ffn_gate.weight", "offset": 12519032832, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.31.ffn_up.weight", "offset": 12635817984, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.31.post_attention_norm.weight", "offset": 12752603136, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 12752621568, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 12752640000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 12752658432, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.31.attn_output.weight", "offset": 12759146496, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.31.attn_q.weight", "offset": 12772122624, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.31.attn_v.weight", "offset": 12785098752, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.32.ffn_gate.weight", "offset": 12792840192, "shape": [ 4608, 36864 ], "size": 7741440, "type": 13 }, { "name": "blk.32.attn_k.weight", "offset": 12909625344, "shape": [ 4608, 2048 ], "size": 116785152, "type": 13 }, { "name": "blk.32.attn_output.weight", "offset": 12916113408, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.32.attn_q.weight", "offset": 12929089536, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.32.attn_v.weight", "offset": 12942065664, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 12948553728, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 12948572160, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.32.ffn_up.weight", "offset": 13065357312, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.32.post_attention_norm.weight", "offset": 13182142464, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 13182160896, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 13182179328, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 13182197760, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 13182216192, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.33.ffn_gate.weight", "offset": 13299001344, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.33.ffn_up.weight", "offset": 13415786496, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.33.post_attention_norm.weight", "offset": 13532571648, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 13532590080, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 13532608512, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 13532626944, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.33.attn_output.weight", "offset": 13539115008, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.33.attn_q.weight", "offset": 13552091136, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.33.attn_v.weight", "offset": 13565067264, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.34.ffn_norm.weight", "offset": 13571555328, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 13571573760, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.34.ffn_gate.weight", "offset": 13710919680, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.34.ffn_up.weight", "offset": 13827704832, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.34.post_attention_norm.weight", "offset": 13944489984, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 13944508416, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_norm.weight", "offset": 13944526848, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 13944545280, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.34.attn_output.weight", "offset": 13951033344, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.34.attn_q.weight", "offset": 13964009472, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.34.attn_v.weight", "offset": 13976985600, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 13984727040, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 13984745472, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.35.ffn_gate.weight", "offset": 14101530624, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.35.ffn_up.weight", "offset": 14218315776, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.35.post_attention_norm.weight", "offset": 14335100928, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 14335119360, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 14335137792, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 14335156224, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.35.attn_output.weight", "offset": 14341644288, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.35.attn_q.weight", "offset": 14354620416, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.35.attn_v.weight", "offset": 14367596544, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 14374084608, "shape": [ 4608, 36864 ], "size": 6488064, "type": 13 }, { "name": "blk.36.ffn_up.weight", "offset": 14490869760, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.36.attn_k.weight", "offset": 14607654912, "shape": [ 4608, 2048 ], "size": 116785152, "type": 13 }, { "name": "blk.36.attn_output.weight", "offset": 14614142976, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.36.attn_q.weight", "offset": 14627119104, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.36.attn_v.weight", "offset": 14640095232, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 14646583296, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 14646601728, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.36.post_attention_norm.weight", "offset": 14763386880, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 14763405312, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 14763423744, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 14763442176, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 14763460608, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.37.ffn_gate.weight", "offset": 14902806528, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.37.ffn_up.weight", "offset": 15019591680, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.37.post_attention_norm.weight", "offset": 15136376832, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 15136395264, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 15136413696, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 15136432128, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.37.attn_output.weight", "offset": 15142920192, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.37.attn_q.weight", "offset": 15155896320, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.37.attn_v.weight", "offset": 15168872448, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 15176613888, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 15176632320, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.38.ffn_gate.weight", "offset": 15293417472, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.38.ffn_up.weight", "offset": 15410202624, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.38.post_attention_norm.weight", "offset": 15526987776, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 15527006208, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 15527024640, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 15527043072, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.38.attn_output.weight", "offset": 15533531136, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.38.attn_q.weight", "offset": 15546507264, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.38.attn_v.weight", "offset": 15559483392, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 15565971456, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 15565989888, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.39.ffn_gate.weight", "offset": 15682775040, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.39.ffn_up.weight", "offset": 15799560192, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.39.post_attention_norm.weight", "offset": 15916345344, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 15916363776, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 15916382208, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 15916400640, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.39.attn_output.weight", "offset": 15922888704, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.39.attn_q.weight", "offset": 15935864832, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.39.attn_v.weight", "offset": 15948840960, "shape": [ 4608, 2048 ], "size": 12976128, "type": 13 }, { "name": "blk.40.ffn_down.weight", "offset": 15955329024, "shape": [ 36864, 4608 ], "size": 6488064, "type": 14 }, { "name": "blk.40.attn_norm.weight", "offset": 16094674944, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.40.ffn_gate.weight", "offset": 16094693376, "shape": [ 4608, 36864 ], "size": 18432, "type": 13 }, { "name": "blk.40.ffn_up.weight", "offset": 16211478528, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.40.post_attention_norm.weight", "offset": 16328263680, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 16328282112, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 16328300544, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 16328318976, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.40.attn_output.weight", "offset": 16334807040, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.40.attn_q.weight", "offset": 16347783168, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.40.attn_v.weight", "offset": 16360759296, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.41.attn_k.weight", "offset": 16368500736, "shape": [ 4608, 2048 ], "size": 7741440, "type": 13 }, { "name": "blk.41.attn_output.weight", "offset": 16374988800, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.41.attn_q.weight", "offset": 16387964928, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.41.attn_v.weight", "offset": 16400941056, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 16408682496, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 16408700928, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.41.ffn_gate.weight", "offset": 16548046848, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.41.ffn_up.weight", "offset": 16664832000, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.41.post_attention_norm.weight", "offset": 16781617152, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 16781635584, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 16781654016, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 16781672448, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 16781690880, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.42.ffn_gate.weight", "offset": 16921036800, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.42.ffn_up.weight", "offset": 17037821952, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.42.post_attention_norm.weight", "offset": 17154607104, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.42.post_ffw_norm.weight", "offset": 17154625536, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 17154643968, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 17154662400, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.42.attn_output.weight", "offset": 17161150464, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.42.attn_q.weight", "offset": 17174126592, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.42.attn_v.weight", "offset": 17187102720, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.43.attn_norm.weight", "offset": 17194844160, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 17194862592, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.43.ffn_gate.weight", "offset": 17334208512, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.43.ffn_up.weight", "offset": 17450993664, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.43.post_attention_norm.weight", "offset": 17567778816, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.43.post_ffw_norm.weight", "offset": 17567797248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 17567815680, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 17567834112, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.43.attn_output.weight", "offset": 17574322176, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.43.attn_q.weight", "offset": 17587298304, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.43.attn_v.weight", "offset": 17600274432, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.44.attn_norm.weight", "offset": 17608015872, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 17608034304, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.44.ffn_gate.weight", "offset": 17747380224, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.44.ffn_up.weight", "offset": 17864165376, "shape": [ 4608, 36864 ], "size": 116785152, "type": 13 }, { "name": "blk.44.post_attention_norm.weight", "offset": 17980950528, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.44.post_ffw_norm.weight", "offset": 17980968960, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 17980987392, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 17981005824, "shape": [ 4608, 2048 ], "size": 18432, "type": 13 }, { "name": "blk.44.attn_output.weight", "offset": 17987493888, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.44.attn_q.weight", "offset": 18000470016, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.44.attn_v.weight", "offset": 18013446144, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.45.ffn_gate.weight", "offset": 18021187584, "shape": [ 4608, 36864 ], "size": 7741440, "type": 13 }, { "name": "blk.45.attn_k.weight", "offset": 18137972736, "shape": [ 4608, 2048 ], "size": 116785152, "type": 13 }, { "name": "blk.45.attn_output.weight", "offset": 18144460800, "shape": [ 4096, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.45.attn_q.weight", "offset": 18157436928, "shape": [ 4608, 4096 ], "size": 12976128, "type": 13 }, { "name": "blk.45.attn_v.weight", "offset": 18170413056, "shape": [ 4608, 2048 ], "size": 12976128, "type": 14 }, { "name": "blk.45.attn_norm.weight", "offset": 18178154496, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 18178172928, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.45.ffn_up.weight", "offset": 18317518848, "shape": [ 4608, 36864 ], "size": 139345920, "type": 13 }, { "name": "blk.45.post_attention_norm.weight", "offset": 18434304000, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.45.post_ffw_norm.weight", "offset": 18434322432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 18434340864, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "output_norm.weight", "offset": 18434359296, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "token_embd.weight", "offset": 18434377728, "shape": [ 4608, 256000 ], "size": 18432, "type": 14 } ], "version": 3 }