gemma2

{ "metadata": { "gemma2.attention.head_count": 32, "gemma2.attention.head_count_kv": 16, "gemma2.attention.key_length": 128, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 128, "gemma2.block_count": 46, "gemma2.context_length": 8192, "gemma2.embedding_length": 4608, "gemma2.feed_forward_length": 36864, "general.architecture": "gemma2", "general.file_type": 14, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 27227128320, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 18432, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 116803584, "shape": [ 4608, 36864 ], "size": 116785152, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 212355072, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.0.post_attention_norm.weight", "offset": 307906560, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 307924992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 307943424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 307961856, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 313270272, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 323887104, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.0.attn_v.weight", "offset": 334503936, "shape": [ 4608, 2048 ], "size": 10616832, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 340992000, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 341010432, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 457795584, "shape": [ 4608, 36864 ], "size": 116785152, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 553347072, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.1.post_attention_norm.weight", "offset": 648898560, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 648916992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 648935424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 648953856, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 654262272, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 664879104, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.1.attn_v.weight", "offset": 675495936, "shape": [ 4608, 2048 ], "size": 10616832, "type": 13 }, { "name": "blk.2.attn_k.weight", "offset": 681984000, "shape": [ 4608, 2048 ], "size": 6488064, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 687292416, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 697909248, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.2.attn_v.weight", "offset": 708526080, "shape": [ 4608, 2048 ], "size": 10616832, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 715014144, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 715032576, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 831817728, "shape": [ 4608, 36864 ], "size": 116785152, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 927369216, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.2.post_attention_norm.weight", "offset": 1022920704, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 1022939136, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 1022957568, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_norm.weight", "offset": 1022976000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1022994432, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 1139779584, "shape": [ 4608, 36864 ], "size": 116785152, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 1235331072, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.3.post_attention_norm.weight", "offset": 1330882560, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 1330900992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 1330919424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1330937856, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 1336246272, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 1346863104, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.3.attn_v.weight", "offset": 1357479936, "shape": [ 4608, 2048 ], "size": 10616832, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 1363968000, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1363986432, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 1480771584, "shape": [ 4608, 36864 ], "size": 116785152, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 1576323072, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.4.post_attention_norm.weight", "offset": 1671874560, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 1671892992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 1671911424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1671929856, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 1677238272, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 1687855104, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.4.attn_v.weight", "offset": 1698471936, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 1703780352, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1703798784, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 1799350272, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 1894901760, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.5.post_attention_norm.weight", "offset": 1990453248, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 1990471680, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 1990490112, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1990508544, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 1995816960, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 2006433792, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.5.attn_v.weight", "offset": 2017050624, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 2022359040, "shape": [ 4608, 36864 ], "size": 5308416, "type": 12 }, { "name": "blk.6.attn_k.weight", "offset": 2117910528, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 2123218944, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 2133835776, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.6.attn_v.weight", "offset": 2144452608, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 2149761024, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2149779456, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 2245330944, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.6.post_attention_norm.weight", "offset": 2340882432, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 2340900864, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 2340919296, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2340937728, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2340956160, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 2436507648, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 2532059136, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.7.post_attention_norm.weight", "offset": 2627610624, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 2627629056, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 2627647488, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2627665920, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 2632974336, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 2643591168, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.7.attn_v.weight", "offset": 2654208000, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 2659516416, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2659534848, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 2755086336, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 2850637824, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.8.post_attention_norm.weight", "offset": 2946189312, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 2946207744, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 2946226176, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2946244608, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 2951553024, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 2962169856, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.8.attn_v.weight", "offset": 2972786688, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 2978095104, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2978113536, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 3073665024, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 3169216512, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.9.post_attention_norm.weight", "offset": 3264768000, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 3264786432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 3264804864, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 3264823296, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 3270131712, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 3280748544, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.9.attn_v.weight", "offset": 3291365376, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.10.ffn_gate.weight", "offset": 3296673792, "shape": [ 4608, 36864 ], "size": 5308416, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 3392225280, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.10.attn_k.weight", "offset": 3487776768, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 3493085184, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.10.attn_q.weight", "offset": 3503702016, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.10.attn_v.weight", "offset": 3514318848, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 3519627264, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 3519645696, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.10.post_attention_norm.weight", "offset": 3615197184, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 3615215616, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 3615234048, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_norm.weight", "offset": 3615252480, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 3615270912, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 3710822400, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 3806373888, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.11.post_attention_norm.weight", "offset": 3901925376, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 3901943808, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 3901962240, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 3901980672, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 3907289088, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.11.attn_q.weight", "offset": 3917905920, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.11.attn_v.weight", "offset": 3928522752, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 3933831168, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 3933849600, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 4029401088, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 4124952576, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.12.post_attention_norm.weight", "offset": 4220504064, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 4220522496, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 4220540928, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 4220559360, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 4225867776, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.12.attn_q.weight", "offset": 4236484608, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.12.attn_v.weight", "offset": 4247101440, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 4252409856, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 4252428288, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 4347979776, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 4443531264, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.13.post_attention_norm.weight", "offset": 4539082752, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 4539101184, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 4539119616, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 4539138048, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 4544446464, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.13.attn_q.weight", "offset": 4555063296, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.13.attn_v.weight", "offset": 4565680128, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 4570988544, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 4571006976, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 4666558464, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 4762109952, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.14.post_attention_norm.weight", "offset": 4857661440, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 4857679872, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 4857698304, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 4857716736, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 4863025152, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 4873641984, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.14.attn_v.weight", "offset": 4884258816, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.15.attn_k.weight", "offset": 4889567232, "shape": [ 4608, 2048 ], "size": 5308416, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 4894875648, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 4905492480, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.15.attn_v.weight", "offset": 4916109312, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 4921417728, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 4921436160, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 5016987648, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 5112539136, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.15.post_attention_norm.weight", "offset": 5208090624, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 5208109056, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 5208127488, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 5208145920, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 5208164352, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 5303715840, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 5399267328, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.16.post_attention_norm.weight", "offset": 5494818816, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 5494837248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 5494855680, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 5494874112, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 5500182528, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.16.attn_q.weight", "offset": 5510799360, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.16.attn_v.weight", "offset": 5521416192, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 5526724608, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 5526743040, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 5622294528, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 5717846016, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.17.post_attention_norm.weight", "offset": 5813397504, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 5813415936, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 5813434368, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 5813452800, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 5818761216, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 5829378048, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.17.attn_v.weight", "offset": 5839994880, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 5845303296, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 5845321728, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 5940873216, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 6036424704, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.18.post_attention_norm.weight", "offset": 6131976192, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 6131994624, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 6132013056, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 6132031488, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 6137339904, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.18.attn_q.weight", "offset": 6147956736, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.18.attn_v.weight", "offset": 6158573568, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 6163881984, "shape": [ 4608, 36864 ], "size": 5308416, "type": 12 }, { "name": "blk.19.attn_k.weight", "offset": 6259433472, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 6264741888, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 6275358720, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.19.attn_v.weight", "offset": 6285975552, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 6291283968, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 6291302400, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 6386853888, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.19.post_attention_norm.weight", "offset": 6482405376, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 6482423808, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 6482442240, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_norm.weight", "offset": 6482460672, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 6482479104, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 6578030592, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 6673582080, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.20.post_attention_norm.weight", "offset": 6769133568, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 6769152000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 6769170432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 6769188864, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 6774497280, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 6785114112, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.20.attn_v.weight", "offset": 6795730944, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 6801039360, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 6801057792, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 6896609280, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 6992160768, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.21.post_attention_norm.weight", "offset": 7087712256, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 7087730688, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 7087749120, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 7087767552, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 7093075968, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 7103692800, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.21.attn_v.weight", "offset": 7114309632, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 7119618048, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 7119636480, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 7215187968, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 7310739456, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.22.post_attention_norm.weight", "offset": 7406290944, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 7406309376, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 7406327808, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 7406346240, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 7411654656, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 7422271488, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.22.attn_v.weight", "offset": 7432888320, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 7438196736, "shape": [ 4608, 36864 ], "size": 5308416, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 7533748224, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.23.attn_k.weight", "offset": 7629299712, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 7634608128, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 7645224960, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.23.attn_v.weight", "offset": 7655841792, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 7661150208, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 7661168640, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.23.post_attention_norm.weight", "offset": 7756720128, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 7756738560, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 7756756992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 7756775424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 7756793856, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 7852345344, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 7947896832, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.24.post_attention_norm.weight", "offset": 8043448320, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 8043466752, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 8043485184, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 8043503616, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 8048812032, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 8059428864, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.24.attn_v.weight", "offset": 8070045696, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 8075354112, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 8075372544, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 8170924032, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 8266475520, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.25.post_attention_norm.weight", "offset": 8362027008, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 8362045440, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 8362063872, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 8362082304, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 8367390720, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 8378007552, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.25.attn_v.weight", "offset": 8388624384, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 8393932800, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 8393951232, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 8489502720, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 8585054208, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.26.post_attention_norm.weight", "offset": 8680605696, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 8680624128, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 8680642560, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 8680660992, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 8685969408, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 8696586240, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.26.attn_v.weight", "offset": 8707203072, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 8712511488, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 8712529920, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 8808081408, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 8903632896, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.27.post_attention_norm.weight", "offset": 8999184384, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 8999202816, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 8999221248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 8999239680, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 9004548096, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.27.attn_q.weight", "offset": 9015164928, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.27.attn_v.weight", "offset": 9025781760, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.28.attn_k.weight", "offset": 9031090176, "shape": [ 4608, 2048 ], "size": 5308416, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 9036398592, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.28.attn_q.weight", "offset": 9047015424, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.28.attn_v.weight", "offset": 9057632256, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 9062940672, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 9062959104, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.28.ffn_gate.weight", "offset": 9158510592, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 9254062080, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.28.post_attention_norm.weight", "offset": 9349613568, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 9349632000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 9349650432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 9349668864, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 9349687296, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.29.ffn_gate.weight", "offset": 9445238784, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 9540790272, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.29.post_attention_norm.weight", "offset": 9636341760, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 9636360192, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 9636378624, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 9636397056, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 9641705472, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.29.attn_q.weight", "offset": 9652322304, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.29.attn_v.weight", "offset": 9662939136, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 9668247552, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 9668265984, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.30.ffn_gate.weight", "offset": 9763817472, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 9859368960, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.30.post_attention_norm.weight", "offset": 9954920448, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 9954938880, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 9954957312, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 9954975744, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 9960284160, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.30.attn_q.weight", "offset": 9970900992, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.30.attn_v.weight", "offset": 9981517824, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 9986826240, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 9986844672, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.31.ffn_gate.weight", "offset": 10082396160, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 10177947648, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.31.post_attention_norm.weight", "offset": 10273499136, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 10273517568, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 10273536000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 10273554432, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.31.attn_output.weight", "offset": 10278862848, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.31.attn_q.weight", "offset": 10289479680, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.31.attn_v.weight", "offset": 10300096512, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.32.ffn_gate.weight", "offset": 10305404928, "shape": [ 4608, 36864 ], "size": 5308416, "type": 12 }, { "name": "blk.32.attn_k.weight", "offset": 10400956416, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.32.attn_output.weight", "offset": 10406264832, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.32.attn_q.weight", "offset": 10416881664, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.32.attn_v.weight", "offset": 10427498496, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 10432806912, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 10432825344, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 10528376832, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.32.post_attention_norm.weight", "offset": 10623928320, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 10623946752, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 10623965184, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 10623983616, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 10624002048, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.33.ffn_gate.weight", "offset": 10719553536, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 10815105024, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.33.post_attention_norm.weight", "offset": 10910656512, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 10910674944, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 10910693376, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 10910711808, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.33.attn_output.weight", "offset": 10916020224, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.33.attn_q.weight", "offset": 10926637056, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.33.attn_v.weight", "offset": 10937253888, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.34.ffn_norm.weight", "offset": 10942562304, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 10942580736, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.34.ffn_gate.weight", "offset": 11038132224, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.34.ffn_up.weight", "offset": 11133683712, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.34.post_attention_norm.weight", "offset": 11229235200, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 11229253632, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_norm.weight", "offset": 11229272064, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 11229290496, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.34.attn_output.weight", "offset": 11234598912, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.34.attn_q.weight", "offset": 11245215744, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.34.attn_v.weight", "offset": 11255832576, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.35.attn_norm.weight", "offset": 11261140992, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 11261159424, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.35.ffn_gate.weight", "offset": 11356710912, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 11452262400, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.35.post_attention_norm.weight", "offset": 11547813888, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 11547832320, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 11547850752, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 11547869184, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 11553177600, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.35.attn_q.weight", "offset": 11563794432, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.35.attn_v.weight", "offset": 11574411264, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.36.ffn_gate.weight", "offset": 11579719680, "shape": [ 4608, 36864 ], "size": 5308416, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 11675271168, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.36.attn_k.weight", "offset": 11770822656, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.36.attn_output.weight", "offset": 11776131072, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.36.attn_q.weight", "offset": 11786747904, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.36.attn_v.weight", "offset": 11797364736, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.36.attn_norm.weight", "offset": 11802673152, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 11802691584, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.36.post_attention_norm.weight", "offset": 11898243072, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 11898261504, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 11898279936, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 11898298368, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 11898316800, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.37.ffn_gate.weight", "offset": 11993868288, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 12089419776, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.37.post_attention_norm.weight", "offset": 12184971264, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 12184989696, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 12185008128, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 12185026560, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.37.attn_output.weight", "offset": 12190334976, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.37.attn_q.weight", "offset": 12200951808, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.37.attn_v.weight", "offset": 12211568640, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 12216877056, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 12216895488, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.38.ffn_gate.weight", "offset": 12312446976, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 12407998464, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.38.post_attention_norm.weight", "offset": 12503549952, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 12503568384, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 12503586816, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 12503605248, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.38.attn_output.weight", "offset": 12508913664, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.38.attn_q.weight", "offset": 12519530496, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.38.attn_v.weight", "offset": 12530147328, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 12535455744, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 12535474176, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.39.ffn_gate.weight", "offset": 12631025664, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 12726577152, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.39.post_attention_norm.weight", "offset": 12822128640, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 12822147072, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 12822165504, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 12822183936, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.39.attn_output.weight", "offset": 12827492352, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.39.attn_q.weight", "offset": 12838109184, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.39.attn_v.weight", "offset": 12848726016, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.40.ffn_down.weight", "offset": 12854034432, "shape": [ 36864, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.40.attn_norm.weight", "offset": 12949585920, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.40.ffn_gate.weight", "offset": 12949604352, "shape": [ 4608, 36864 ], "size": 18432, "type": 12 }, { "name": "blk.40.ffn_up.weight", "offset": 13045155840, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.40.post_attention_norm.weight", "offset": 13140707328, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 13140725760, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 13140744192, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 13140762624, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.40.attn_output.weight", "offset": 13146071040, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.40.attn_q.weight", "offset": 13156687872, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.40.attn_v.weight", "offset": 13167304704, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.41.attn_k.weight", "offset": 13172613120, "shape": [ 4608, 2048 ], "size": 5308416, "type": 12 }, { "name": "blk.41.attn_output.weight", "offset": 13177921536, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.41.attn_q.weight", "offset": 13188538368, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.41.attn_v.weight", "offset": 13199155200, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.41.attn_norm.weight", "offset": 13204463616, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 13204482048, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.41.ffn_gate.weight", "offset": 13300033536, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.41.ffn_up.weight", "offset": 13395585024, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.41.post_attention_norm.weight", "offset": 13491136512, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 13491154944, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 13491173376, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 13491191808, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 13491210240, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.42.ffn_gate.weight", "offset": 13586761728, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.42.ffn_up.weight", "offset": 13682313216, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.42.post_attention_norm.weight", "offset": 13777864704, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.42.post_ffw_norm.weight", "offset": 13777883136, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 13777901568, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 13777920000, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.42.attn_output.weight", "offset": 13783228416, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.42.attn_q.weight", "offset": 13793845248, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.42.attn_v.weight", "offset": 13804462080, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.43.attn_norm.weight", "offset": 13809770496, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 13809788928, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.43.ffn_gate.weight", "offset": 13905340416, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.43.ffn_up.weight", "offset": 14000891904, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.43.post_attention_norm.weight", "offset": 14096443392, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.43.post_ffw_norm.weight", "offset": 14096461824, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 14096480256, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 14096498688, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.43.attn_output.weight", "offset": 14101807104, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.43.attn_q.weight", "offset": 14112423936, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.43.attn_v.weight", "offset": 14123040768, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.44.attn_norm.weight", "offset": 14128349184, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 14128367616, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.44.ffn_gate.weight", "offset": 14223919104, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.44.ffn_up.weight", "offset": 14319470592, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.44.post_attention_norm.weight", "offset": 14415022080, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.44.post_ffw_norm.weight", "offset": 14415040512, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 14415058944, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 14415077376, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.44.attn_output.weight", "offset": 14420385792, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.44.attn_q.weight", "offset": 14431002624, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.44.attn_v.weight", "offset": 14441619456, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.45.ffn_gate.weight", "offset": 14446927872, "shape": [ 4608, 36864 ], "size": 5308416, "type": 12 }, { "name": "blk.45.attn_k.weight", "offset": 14542479360, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.45.attn_output.weight", "offset": 14547787776, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.45.attn_q.weight", "offset": 14558404608, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.45.attn_v.weight", "offset": 14569021440, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.45.attn_norm.weight", "offset": 14574329856, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 14574348288, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.45.ffn_up.weight", "offset": 14669899776, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.45.post_attention_norm.weight", "offset": 14765451264, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.45.post_ffw_norm.weight", "offset": 14765469696, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 14765488128, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "output_norm.weight", "offset": 14765506560, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "token_embd.weight", "offset": 14765524992, "shape": [ 4608, 256000 ], "size": 18432, "type": 14 } ], "version": 3 }