gemma2

{ "metadata": { "gemma2.attention.head_count": 32, "gemma2.attention.head_count_kv": 16, "gemma2.attention.key_length": 128, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 128, "gemma2.block_count": 46, "gemma2.context_length": 8192, "gemma2.embedding_length": 4608, "gemma2.feed_forward_length": 36864, "general.architecture": "gemma2", "general.file_type": 15, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 27227128320, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 18432, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 139364352, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 234915840, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.0.post_attention_norm.weight", "offset": 330467328, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 330485760, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 330504192, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 330522624, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 335831040, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 346447872, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.0.attn_v.weight", "offset": 357064704, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 364806144, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 364824576, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 504170496, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 599721984, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.1.post_attention_norm.weight", "offset": 695273472, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 695291904, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 695310336, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 695328768, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 700637184, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 711254016, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.1.attn_v.weight", "offset": 721870848, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.2.attn_k.weight", "offset": 729612288, "shape": [ 4608, 2048 ], "size": 7741440, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 734920704, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 745537536, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.2.attn_v.weight", "offset": 756154368, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 763895808, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 763914240, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 903260160, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 998811648, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.2.post_attention_norm.weight", "offset": 1094363136, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 1094381568, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 1094400000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_norm.weight", "offset": 1094418432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1094436864, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 1233782784, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 1329334272, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.3.post_attention_norm.weight", "offset": 1424885760, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 1424904192, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 1424922624, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1424941056, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 1430249472, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 1440866304, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.3.attn_v.weight", "offset": 1451483136, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1459224576, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1459243008, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 1598588928, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 1694140416, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.4.post_attention_norm.weight", "offset": 1789691904, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 1789710336, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 1789728768, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1789747200, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 1795055616, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 1805672448, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.4.attn_v.weight", "offset": 1816289280, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 1824030720, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1824049152, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 1919600640, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 2015152128, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.5.post_attention_norm.weight", "offset": 2110703616, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 2110722048, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 2110740480, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 2110758912, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 2116067328, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 2126684160, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.5.attn_v.weight", "offset": 2137300992, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 2142609408, "shape": [ 4608, 36864 ], "size": 5308416, "type": 12 }, { "name": "blk.6.attn_k.weight", "offset": 2238160896, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 2243469312, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 2254086144, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.6.attn_v.weight", "offset": 2264702976, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 2270011392, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2270029824, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 2365581312, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.6.post_attention_norm.weight", "offset": 2461132800, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 2461151232, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 2461169664, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2461188096, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2461206528, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 2600552448, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 2696103936, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.7.post_attention_norm.weight", "offset": 2791655424, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 2791673856, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 2791692288, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2791710720, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 2797019136, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 2807635968, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.7.attn_v.weight", "offset": 2818252800, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.8.attn_norm.weight", "offset": 2825994240, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2826012672, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 2921564160, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 3017115648, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.8.post_attention_norm.weight", "offset": 3112667136, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 3112685568, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 3112704000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 3112722432, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 3118030848, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 3128647680, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.8.attn_v.weight", "offset": 3139264512, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 3144572928, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3144591360, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 3240142848, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 3335694336, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.9.post_attention_norm.weight", "offset": 3431245824, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 3431264256, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 3431282688, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 3431301120, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 3436609536, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 3447226368, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.9.attn_v.weight", "offset": 3457843200, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.10.ffn_gate.weight", "offset": 3463151616, "shape": [ 4608, 36864 ], "size": 5308416, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 3558703104, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.10.attn_k.weight", "offset": 3654254592, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 3659563008, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.10.attn_q.weight", "offset": 3670179840, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.10.attn_v.weight", "offset": 3680796672, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 3688538112, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 3688556544, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.10.post_attention_norm.weight", "offset": 3827902464, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 3827920896, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 3827939328, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_norm.weight", "offset": 3827957760, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 3827976192, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 3923527680, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 4019079168, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.11.post_attention_norm.weight", "offset": 4114630656, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 4114649088, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 4114667520, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 4114685952, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 4119994368, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.11.attn_q.weight", "offset": 4130611200, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.11.attn_v.weight", "offset": 4141228032, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 4146536448, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 4146554880, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 4242106368, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 4337657856, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.12.post_attention_norm.weight", "offset": 4433209344, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 4433227776, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 4433246208, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 4433264640, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 4438573056, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.12.attn_q.weight", "offset": 4449189888, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.12.attn_v.weight", "offset": 4459806720, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 4465115136, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 4465133568, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.13.ffn_gate.weight", "offset": 4604479488, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 4700030976, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.13.post_attention_norm.weight", "offset": 4795582464, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 4795600896, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 4795619328, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 4795637760, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 4800946176, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.13.attn_q.weight", "offset": 4811563008, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.13.attn_v.weight", "offset": 4822179840, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 4829921280, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 4829939712, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 4925491200, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 5021042688, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.14.post_attention_norm.weight", "offset": 5116594176, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 5116612608, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 5116631040, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 5116649472, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 5121957888, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 5132574720, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.14.attn_v.weight", "offset": 5143191552, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.15.attn_k.weight", "offset": 5148499968, "shape": [ 4608, 2048 ], "size": 5308416, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 5153808384, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 5164425216, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.15.attn_v.weight", "offset": 5175042048, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 5180350464, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 5180368896, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 5275920384, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 5371471872, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.15.post_attention_norm.weight", "offset": 5467023360, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 5467041792, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 5467060224, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 5467078656, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 5467097088, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 5606443008, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 5701994496, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.16.post_attention_norm.weight", "offset": 5797545984, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 5797564416, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 5797582848, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 5797601280, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 5802909696, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.16.attn_q.weight", "offset": 5813526528, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.16.attn_v.weight", "offset": 5824143360, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 5831884800, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 5831903232, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 5927454720, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 6023006208, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.17.post_attention_norm.weight", "offset": 6118557696, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 6118576128, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 6118594560, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 6118612992, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 6123921408, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 6134538240, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.17.attn_v.weight", "offset": 6145155072, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 6150463488, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 6150481920, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 6246033408, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 6341584896, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.18.post_attention_norm.weight", "offset": 6437136384, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 6437154816, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 6437173248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 6437191680, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 6442500096, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.18.attn_q.weight", "offset": 6453116928, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.18.attn_v.weight", "offset": 6463733760, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 6469042176, "shape": [ 4608, 36864 ], "size": 5308416, "type": 12 }, { "name": "blk.19.attn_k.weight", "offset": 6564593664, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 6569902080, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 6580518912, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.19.attn_v.weight", "offset": 6591135744, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 6598877184, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 6598895616, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 6738241536, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.19.post_attention_norm.weight", "offset": 6833793024, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 6833811456, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 6833829888, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_norm.weight", "offset": 6833848320, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 6833866752, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 6929418240, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 7024969728, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.20.post_attention_norm.weight", "offset": 7120521216, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 7120539648, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 7120558080, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 7120576512, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 7125884928, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 7136501760, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.20.attn_v.weight", "offset": 7147118592, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 7152427008, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 7152445440, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 7247996928, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 7343548416, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.21.post_attention_norm.weight", "offset": 7439099904, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 7439118336, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 7439136768, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 7439155200, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 7444463616, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 7455080448, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.21.attn_v.weight", "offset": 7465697280, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 7471005696, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 7471024128, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.22.ffn_gate.weight", "offset": 7610370048, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 7705921536, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.22.post_attention_norm.weight", "offset": 7801473024, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 7801491456, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 7801509888, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 7801528320, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 7806836736, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 7817453568, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.22.attn_v.weight", "offset": 7828070400, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.23.ffn_up.weight", "offset": 7835811840, "shape": [ 4608, 36864 ], "size": 7741440, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 7931363328, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.23.attn_k.weight", "offset": 8026914816, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 8032223232, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 8042840064, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.23.attn_v.weight", "offset": 8053456896, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 8058765312, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 8058783744, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.23.post_attention_norm.weight", "offset": 8154335232, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 8154353664, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 8154372096, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 8154390528, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 8154408960, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 8249960448, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 8345511936, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.24.post_attention_norm.weight", "offset": 8441063424, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 8441081856, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 8441100288, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 8441118720, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 8446427136, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 8457043968, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.24.attn_v.weight", "offset": 8467660800, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 8472969216, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 8472987648, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 8612333568, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 8707885056, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.25.post_attention_norm.weight", "offset": 8803436544, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 8803454976, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 8803473408, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 8803491840, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 8808800256, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 8819417088, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.25.attn_v.weight", "offset": 8830033920, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 8837775360, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 8837793792, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 8933345280, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 9028896768, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.26.post_attention_norm.weight", "offset": 9124448256, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 9124466688, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 9124485120, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 9124503552, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 9129811968, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 9140428800, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.26.attn_v.weight", "offset": 9151045632, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 9156354048, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 9156372480, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 9251923968, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 9347475456, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.27.post_attention_norm.weight", "offset": 9443026944, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 9443045376, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 9443063808, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 9443082240, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 9448390656, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.27.attn_q.weight", "offset": 9459007488, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.27.attn_v.weight", "offset": 9469624320, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.28.attn_k.weight", "offset": 9474932736, "shape": [ 4608, 2048 ], "size": 5308416, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 9480241152, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.28.attn_q.weight", "offset": 9490857984, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.28.attn_v.weight", "offset": 9501474816, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.28.attn_norm.weight", "offset": 9509216256, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 9509234688, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.28.ffn_gate.weight", "offset": 9648580608, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 9744132096, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.28.post_attention_norm.weight", "offset": 9839683584, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 9839702016, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 9839720448, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 9839738880, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 9839757312, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.29.ffn_gate.weight", "offset": 9935308800, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 10030860288, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.29.post_attention_norm.weight", "offset": 10126411776, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 10126430208, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 10126448640, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 10126467072, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 10131775488, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.29.attn_q.weight", "offset": 10142392320, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.29.attn_v.weight", "offset": 10153009152, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 10158317568, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 10158336000, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.30.ffn_gate.weight", "offset": 10253887488, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 10349438976, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.30.post_attention_norm.weight", "offset": 10444990464, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 10445008896, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 10445027328, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 10445045760, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 10450354176, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.30.attn_q.weight", "offset": 10460971008, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.30.attn_v.weight", "offset": 10471587840, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 10476896256, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 10476914688, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.31.ffn_gate.weight", "offset": 10616260608, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 10711812096, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.31.post_attention_norm.weight", "offset": 10807363584, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 10807382016, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 10807400448, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 10807418880, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.31.attn_output.weight", "offset": 10812727296, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.31.attn_q.weight", "offset": 10823344128, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.31.attn_v.weight", "offset": 10833960960, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.32.ffn_gate.weight", "offset": 10841702400, "shape": [ 4608, 36864 ], "size": 7741440, "type": 12 }, { "name": "blk.32.attn_k.weight", "offset": 10937253888, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.32.attn_output.weight", "offset": 10942562304, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.32.attn_q.weight", "offset": 10953179136, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.32.attn_v.weight", "offset": 10963795968, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 10969104384, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 10969122816, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 11064674304, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.32.post_attention_norm.weight", "offset": 11160225792, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 11160244224, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 11160262656, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 11160281088, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 11160299520, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.33.ffn_gate.weight", "offset": 11255851008, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 11351402496, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.33.post_attention_norm.weight", "offset": 11446953984, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 11446972416, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 11446990848, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 11447009280, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.33.attn_output.weight", "offset": 11452317696, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.33.attn_q.weight", "offset": 11462934528, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.33.attn_v.weight", "offset": 11473551360, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.34.ffn_norm.weight", "offset": 11478859776, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 11478878208, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.34.ffn_gate.weight", "offset": 11618224128, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.34.ffn_up.weight", "offset": 11713775616, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.34.post_attention_norm.weight", "offset": 11809327104, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 11809345536, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_norm.weight", "offset": 11809363968, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 11809382400, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.34.attn_output.weight", "offset": 11814690816, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.34.attn_q.weight", "offset": 11825307648, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.34.attn_v.weight", "offset": 11835924480, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 11843665920, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 11843684352, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.35.ffn_gate.weight", "offset": 11939235840, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 12034787328, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.35.post_attention_norm.weight", "offset": 12130338816, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 12130357248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 12130375680, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 12130394112, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 12135702528, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.35.attn_q.weight", "offset": 12146319360, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.35.attn_v.weight", "offset": 12156936192, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.36.ffn_gate.weight", "offset": 12162244608, "shape": [ 4608, 36864 ], "size": 5308416, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 12257796096, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.36.attn_k.weight", "offset": 12353347584, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.36.attn_output.weight", "offset": 12358656000, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.36.attn_q.weight", "offset": 12369272832, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.36.attn_v.weight", "offset": 12379889664, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.36.attn_norm.weight", "offset": 12385198080, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 12385216512, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.36.post_attention_norm.weight", "offset": 12480768000, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 12480786432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 12480804864, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 12480823296, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 12480841728, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.37.ffn_gate.weight", "offset": 12620187648, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 12715739136, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.37.post_attention_norm.weight", "offset": 12811290624, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 12811309056, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 12811327488, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 12811345920, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.37.attn_output.weight", "offset": 12816654336, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.37.attn_q.weight", "offset": 12827271168, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.37.attn_v.weight", "offset": 12837888000, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 12845629440, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 12845647872, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.38.ffn_gate.weight", "offset": 12941199360, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 13036750848, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.38.post_attention_norm.weight", "offset": 13132302336, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 13132320768, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 13132339200, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 13132357632, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.38.attn_output.weight", "offset": 13137666048, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.38.attn_q.weight", "offset": 13148282880, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.38.attn_v.weight", "offset": 13158899712, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 13164208128, "shape": [ 4608 ], "size": 5308416, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 13164226560, "shape": [ 36864, 4608 ], "size": 18432, "type": 12 }, { "name": "blk.39.ffn_gate.weight", "offset": 13259778048, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 13355329536, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.39.post_attention_norm.weight", "offset": 13450881024, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 13450899456, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 13450917888, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 13450936320, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.39.attn_output.weight", "offset": 13456244736, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.39.attn_q.weight", "offset": 13466861568, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.39.attn_v.weight", "offset": 13477478400, "shape": [ 4608, 2048 ], "size": 10616832, "type": 12 }, { "name": "blk.40.ffn_down.weight", "offset": 13482786816, "shape": [ 36864, 4608 ], "size": 5308416, "type": 14 }, { "name": "blk.40.attn_norm.weight", "offset": 13622132736, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.40.ffn_gate.weight", "offset": 13622151168, "shape": [ 4608, 36864 ], "size": 18432, "type": 12 }, { "name": "blk.40.ffn_up.weight", "offset": 13717702656, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.40.post_attention_norm.weight", "offset": 13813254144, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 13813272576, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 13813291008, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 13813309440, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.40.attn_output.weight", "offset": 13818617856, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.40.attn_q.weight", "offset": 13829234688, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.40.attn_v.weight", "offset": 13839851520, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.41.attn_k.weight", "offset": 13847592960, "shape": [ 4608, 2048 ], "size": 7741440, "type": 12 }, { "name": "blk.41.attn_output.weight", "offset": 13852901376, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.41.attn_q.weight", "offset": 13863518208, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.41.attn_v.weight", "offset": 13874135040, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 13881876480, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 13881894912, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.41.ffn_gate.weight", "offset": 14021240832, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.41.ffn_up.weight", "offset": 14116792320, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.41.post_attention_norm.weight", "offset": 14212343808, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 14212362240, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 14212380672, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 14212399104, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 14212417536, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.42.ffn_gate.weight", "offset": 14351763456, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.42.ffn_up.weight", "offset": 14447314944, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.42.post_attention_norm.weight", "offset": 14542866432, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.42.post_ffw_norm.weight", "offset": 14542884864, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 14542903296, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 14542921728, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.42.attn_output.weight", "offset": 14548230144, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.42.attn_q.weight", "offset": 14558846976, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.42.attn_v.weight", "offset": 14569463808, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.43.attn_norm.weight", "offset": 14577205248, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 14577223680, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.43.ffn_gate.weight", "offset": 14716569600, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.43.ffn_up.weight", "offset": 14812121088, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.43.post_attention_norm.weight", "offset": 14907672576, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.43.post_ffw_norm.weight", "offset": 14907691008, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 14907709440, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 14907727872, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.43.attn_output.weight", "offset": 14913036288, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.43.attn_q.weight", "offset": 14923653120, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.43.attn_v.weight", "offset": 14934269952, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.44.attn_norm.weight", "offset": 14942011392, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 14942029824, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.44.ffn_gate.weight", "offset": 15081375744, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.44.ffn_up.weight", "offset": 15176927232, "shape": [ 4608, 36864 ], "size": 95551488, "type": 12 }, { "name": "blk.44.post_attention_norm.weight", "offset": 15272478720, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.44.post_ffw_norm.weight", "offset": 15272497152, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 15272515584, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 15272534016, "shape": [ 4608, 2048 ], "size": 18432, "type": 12 }, { "name": "blk.44.attn_output.weight", "offset": 15277842432, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.44.attn_q.weight", "offset": 15288459264, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.44.attn_v.weight", "offset": 15299076096, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.45.ffn_gate.weight", "offset": 15306817536, "shape": [ 4608, 36864 ], "size": 7741440, "type": 12 }, { "name": "blk.45.attn_k.weight", "offset": 15402369024, "shape": [ 4608, 2048 ], "size": 95551488, "type": 12 }, { "name": "blk.45.attn_output.weight", "offset": 15407677440, "shape": [ 4096, 4608 ], "size": 5308416, "type": 12 }, { "name": "blk.45.attn_q.weight", "offset": 15418294272, "shape": [ 4608, 4096 ], "size": 10616832, "type": 12 }, { "name": "blk.45.attn_v.weight", "offset": 15428911104, "shape": [ 4608, 2048 ], "size": 10616832, "type": 14 }, { "name": "blk.45.attn_norm.weight", "offset": 15436652544, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 15436670976, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.45.ffn_up.weight", "offset": 15576016896, "shape": [ 4608, 36864 ], "size": 139345920, "type": 12 }, { "name": "blk.45.post_attention_norm.weight", "offset": 15671568384, "shape": [ 4608 ], "size": 95551488, "type": 0 }, { "name": "blk.45.post_ffw_norm.weight", "offset": 15671586816, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 15671605248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "output_norm.weight", "offset": 15671623680, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "token_embd.weight", "offset": 15671642112, "shape": [ 4608, 256000 ], "size": 18432, "type": 14 } ], "version": 3 }