gemma2

{ "metadata": { "gemma2.attention.head_count": 16, "gemma2.attention.head_count_kv": 8, "gemma2.attention.key_length": 256, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 256, "gemma2.block_count": 42, "gemma2.context_length": 8192, "gemma2.embedding_length": 3584, "gemma2.feed_forward_length": 14336, "general.architecture": "gemma2", "general.file_type": 9, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 9241705984, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 14336, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.0.ffn_gate.weight", "offset": 38549504, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 77084672, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.0.post_attention_norm.weight", "offset": 115619840, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 115634176, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 115648512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 115662848, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.0.attn_output.weight", "offset": 121167872, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.0.attn_q.weight", "offset": 132177920, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.0.attn_v.weight", "offset": 143187968, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 148692992, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 148707328, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.1.ffn_gate.weight", "offset": 187242496, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.1.ffn_up.weight", "offset": 225777664, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.1.post_attention_norm.weight", "offset": 264312832, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 264327168, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 264341504, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 264355840, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.1.attn_output.weight", "offset": 269860864, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.1.attn_q.weight", "offset": 280870912, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.1.attn_v.weight", "offset": 291880960, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.2.attn_v.weight", "offset": 297385984, "shape": [ 3584, 2048 ], "size": 5505024, "type": 7 }, { "name": "blk.2.ffn_down.weight", "offset": 302891008, "shape": [ 14336, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.2.ffn_gate.weight", "offset": 341426176, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.2.ffn_up.weight", "offset": 379961344, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.2.post_attention_norm.weight", "offset": 418496512, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 418510848, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 418525184, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 418539520, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.2.attn_output.weight", "offset": 424044544, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.2.attn_q.weight", "offset": 435054592, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 446064640, "shape": [ 3584 ], "size": 11010048, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 446078976, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.3.ffn_down.weight", "offset": 451584000, "shape": [ 14336, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.3.ffn_gate.weight", "offset": 490119168, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.3.ffn_up.weight", "offset": 528654336, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.3.post_attention_norm.weight", "offset": 567189504, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 567203840, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 567218176, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 567232512, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.3.attn_output.weight", "offset": 572737536, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.3.attn_q.weight", "offset": 583747584, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 594757632, "shape": [ 3584 ], "size": 11010048, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 594771968, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 594786304, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.4.ffn_gate.weight", "offset": 633321472, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.4.ffn_up.weight", "offset": 671856640, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.4.post_attention_norm.weight", "offset": 710391808, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 710406144, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 710420480, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 710434816, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.4.attn_output.weight", "offset": 715939840, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.4.attn_q.weight", "offset": 726949888, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.4.attn_v.weight", "offset": 737959936, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 743464960, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 743479296, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.5.ffn_gate.weight", "offset": 782014464, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.5.ffn_up.weight", "offset": 820549632, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.5.post_attention_norm.weight", "offset": 859084800, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 859099136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 859113472, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 859127808, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.5.attn_output.weight", "offset": 864632832, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.5.attn_q.weight", "offset": 875642880, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.5.attn_v.weight", "offset": 886652928, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 892157952, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 892172288, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.6.ffn_gate.weight", "offset": 930707456, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.6.ffn_up.weight", "offset": 969242624, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.6.post_attention_norm.weight", "offset": 1007777792, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 1007792128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 1007806464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1007820800, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 1013325824, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.6.attn_q.weight", "offset": 1024335872, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.6.attn_v.weight", "offset": 1035345920, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.7.attn_norm.weight", "offset": 1040850944, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 1040865280, "shape": [ 3584, 14336 ], "size": 14336, "type": 7 }, { "name": "blk.7.attn_k.weight", "offset": 1079400448, "shape": [ 3584, 2048 ], "size": 38535168, "type": 7 }, { "name": "blk.7.attn_output.weight", "offset": 1084905472, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.7.attn_q.weight", "offset": 1095915520, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.7.attn_v.weight", "offset": 1106925568, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.7.ffn_gate.weight", "offset": 1112430592, "shape": [ 3584, 14336 ], "size": 5505024, "type": 7 }, { "name": "blk.7.ffn_down.weight", "offset": 1150965760, "shape": [ 14336, 3584 ], "size": 38535168, "type": 7 }, { "name": "blk.7.post_attention_norm.weight", "offset": 1189500928, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 1189515264, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 1189529600, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 1189543936, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1189558272, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.8.ffn_gate.weight", "offset": 1228093440, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.8.ffn_up.weight", "offset": 1266628608, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 1305163776, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 1305178112, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1305192448, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.8.attn_output.weight", "offset": 1310697472, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.8.attn_q.weight", "offset": 1321707520, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.8.attn_v.weight", "offset": 1332717568, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.8.post_attention_norm.weight", "offset": 1338222592, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1338236928, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.9.ffn_down.weight", "offset": 1343741952, "shape": [ 14336, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 1382277120, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.9.ffn_gate.weight", "offset": 1382291456, "shape": [ 3584, 14336 ], "size": 14336, "type": 7 }, { "name": "blk.9.ffn_up.weight", "offset": 1420826624, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.9.post_attention_norm.weight", "offset": 1459361792, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 1459376128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 1459390464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1459404800, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.9.attn_output.weight", "offset": 1464909824, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.9.attn_q.weight", "offset": 1475919872, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.10.attn_q.weight", "offset": 1486929920, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.10.attn_v.weight", "offset": 1497939968, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.10.ffn_down.weight", "offset": 1503444992, "shape": [ 14336, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.10.ffn_gate.weight", "offset": 1541980160, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.10.attn_k.weight", "offset": 1580515328, "shape": [ 3584, 2048 ], "size": 38535168, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 1586020352, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 1586034688, "shape": [ 3584, 14336 ], "size": 14336, "type": 7 }, { "name": "blk.10.post_attention_norm.weight", "offset": 1624569856, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 1624584192, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1624598528, "shape": [ 4096, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 1635608576, "shape": [ 3584 ], "size": 11010048, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1635622912, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.post_attention_norm.weight", "offset": 1635637248, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1635651584, "shape": [ 3584, 4096 ], "size": 14336, "type": 7 }, { "name": "blk.11.ffn_up.weight", "offset": 1646661632, "shape": [ 3584, 14336 ], "size": 11010048, "type": 7 }, { "name": "blk.11.attn_k.weight", "offset": 1685196800, "shape": [ 3584, 2048 ], "size": 38535168, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 1690701824, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 1690716160, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1690730496, "shape": [ 4096, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.11.attn_v.weight", "offset": 1701740544, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.11.ffn_down.weight", "offset": 1707245568, "shape": [ 14336, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.11.ffn_gate.weight", "offset": 1745780736, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 1784315904, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1784330240, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.12.attn_v.weight", "offset": 1822865408, "shape": [ 3584, 2048 ], "size": 38535168, "type": 7 }, { "name": "blk.12.attn_q.weight", "offset": 1828370432, "shape": [ 3584, 4096 ], "size": 5505024, "type": 7 }, { "name": "blk.12.attn_output.weight", "offset": 1839380480, "shape": [ 4096, 3584 ], "size": 11010048, "type": 7 }, { "name": "blk.12.attn_k.weight", "offset": 1850390528, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 1855895552, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 1855909888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.post_attention_norm.weight", "offset": 1855924224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 1855938560, "shape": [ 3584, 14336 ], "size": 14336, "type": 7 }, { "name": "blk.12.ffn_gate.weight", "offset": 1894473728, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.13.attn_output.weight", "offset": 1933008896, "shape": [ 4096, 3584 ], "size": 38535168, "type": 7 }, { "name": "blk.13.attn_v.weight", "offset": 1944018944, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 1949523968, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 1949538304, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.post_attention_norm.weight", "offset": 1949552640, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 1949566976, "shape": [ 3584, 14336 ], "size": 14336, "type": 7 }, { "name": "blk.13.ffn_gate.weight", "offset": 1988102144, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.13.ffn_down.weight", "offset": 2026637312, "shape": [ 14336, 3584 ], "size": 38535168, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 2065172480, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2065186816, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.13.attn_q.weight", "offset": 2070691840, "shape": [ 3584, 4096 ], "size": 5505024, "type": 7 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 2081701888, "shape": [ 3584 ], "size": 11010048, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 2081716224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2081730560, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.14.ffn_gate.weight", "offset": 2120265728, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.14.ffn_up.weight", "offset": 2158800896, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.14.post_attention_norm.weight", "offset": 2197336064, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2197350400, "shape": [ 4096, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 2208360448, "shape": [ 3584 ], "size": 11010048, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2208374784, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.14.attn_v.weight", "offset": 2213879808, "shape": [ 3584, 2048 ], "size": 5505024, "type": 7 }, { "name": "blk.14.attn_q.weight", "offset": 2219384832, "shape": [ 3584, 4096 ], "size": 5505024, "type": 7 }, { "name": "blk.15.attn_v.weight", "offset": 2230394880, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.15.attn_q.weight", "offset": 2235899904, "shape": [ 3584, 4096 ], "size": 5505024, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 2246909952, "shape": [ 3584 ], "size": 11010048, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2246924288, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.15.ffn_gate.weight", "offset": 2285459456, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.15.ffn_up.weight", "offset": 2323994624, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.15.post_attention_norm.weight", "offset": 2362529792, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 2362544128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 2362558464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2362572800, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.15.attn_output.weight", "offset": 2368077824, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 2379087872, "shape": [ 3584 ], "size": 11010048, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 2379102208, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2379116544, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 2384621568, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 2384635904, "shape": [ 3584, 4096 ], "size": 14336, "type": 7 }, { "name": "blk.16.attn_v.weight", "offset": 2395645952, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.16.attn_output.weight", "offset": 2401150976, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.16.post_attention_norm.weight", "offset": 2412161024, "shape": [ 3584 ], "size": 11010048, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 2412175360, "shape": [ 3584, 14336 ], "size": 14336, "type": 7 }, { "name": "blk.16.ffn_gate.weight", "offset": 2450710528, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.16.ffn_down.weight", "offset": 2489245696, "shape": [ 14336, 3584 ], "size": 38535168, "type": 7 }, { "name": "blk.17.attn_norm.weight", "offset": 2527780864, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 2527795200, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2527809536, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.17.attn_output.weight", "offset": 2533314560, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 2544324608, "shape": [ 3584 ], "size": 11010048, "type": 0 }, { "name": "blk.17.post_attention_norm.weight", "offset": 2544338944, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 2544353280, "shape": [ 3584, 14336 ], "size": 14336, "type": 7 }, { "name": "blk.17.ffn_gate.weight", "offset": 2582888448, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.17.ffn_down.weight", "offset": 2621423616, "shape": [ 14336, 3584 ], "size": 38535168, "type": 7 }, { "name": "blk.17.attn_v.weight", "offset": 2659958784, "shape": [ 3584, 2048 ], "size": 38535168, "type": 7 }, { "name": "blk.17.attn_q.weight", "offset": 2665463808, "shape": [ 3584, 4096 ], "size": 5505024, "type": 7 }, { "name": "blk.18.attn_v.weight", "offset": 2676473856, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 2681978880, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 2681993216, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2682007552, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.18.ffn_gate.weight", "offset": 2720542720, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.18.ffn_up.weight", "offset": 2759077888, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.18.post_attention_norm.weight", "offset": 2797613056, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2797627392, "shape": [ 3584, 4096 ], "size": 14336, "type": 7 }, { "name": "blk.18.attn_output.weight", "offset": 2808637440, "shape": [ 4096, 3584 ], "size": 11010048, "type": 7 }, { "name": "blk.18.attn_k.weight", "offset": 2819647488, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 2825152512, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 2825166848, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 2825181184, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2825195520, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.19.ffn_gate.weight", "offset": 2863730688, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.19.ffn_up.weight", "offset": 2902265856, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.19.post_attention_norm.weight", "offset": 2940801024, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 2940815360, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2940829696, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.19.attn_output.weight", "offset": 2946334720, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.19.attn_q.weight", "offset": 2957344768, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.19.attn_v.weight", "offset": 2968354816, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.20.attn_k.weight", "offset": 2973859840, "shape": [ 3584, 2048 ], "size": 5505024, "type": 7 }, { "name": "blk.20.attn_output.weight", "offset": 2979364864, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.20.attn_q.weight", "offset": 2990374912, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.20.attn_v.weight", "offset": 3001384960, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 3006889984, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3006904320, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.20.ffn_gate.weight", "offset": 3045439488, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.20.ffn_up.weight", "offset": 3083974656, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.20.post_attention_norm.weight", "offset": 3122509824, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 3122524160, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 3122538496, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3122552832, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 3161088000, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.21.ffn_gate.weight", "offset": 3161102336, "shape": [ 3584, 14336 ], "size": 14336, "type": 7 }, { "name": "blk.21.ffn_up.weight", "offset": 3199637504, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.21.post_attention_norm.weight", "offset": 3238172672, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 3238187008, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 3238201344, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 3238215680, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 3243720704, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.21.attn_q.weight", "offset": 3254730752, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.21.attn_v.weight", "offset": 3265740800, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.22.attn_norm.weight", "offset": 3271245824, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3271260160, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.22.ffn_gate.weight", "offset": 3309795328, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.22.ffn_up.weight", "offset": 3348330496, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.22.post_attention_norm.weight", "offset": 3386865664, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 3386880000, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 3386894336, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3386908672, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.22.attn_output.weight", "offset": 3392413696, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.22.attn_q.weight", "offset": 3403423744, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.22.attn_v.weight", "offset": 3414433792, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 3419938816, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3419953152, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.23.ffn_gate.weight", "offset": 3458488320, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.23.ffn_up.weight", "offset": 3497023488, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.23.post_attention_norm.weight", "offset": 3535558656, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 3535572992, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 3535587328, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3535601664, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.23.attn_output.weight", "offset": 3541106688, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.23.attn_q.weight", "offset": 3552116736, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.23.attn_v.weight", "offset": 3563126784, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.24.attn_norm.weight", "offset": 3568631808, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3568646144, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.24.ffn_gate.weight", "offset": 3607181312, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.24.ffn_up.weight", "offset": 3645716480, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.24.post_attention_norm.weight", "offset": 3684251648, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 3684265984, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 3684280320, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 3684294656, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.24.attn_output.weight", "offset": 3689799680, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.24.attn_q.weight", "offset": 3700809728, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.24.attn_v.weight", "offset": 3711819776, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.25.attn_norm.weight", "offset": 3717324800, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3717339136, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.25.ffn_gate.weight", "offset": 3755874304, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.25.ffn_up.weight", "offset": 3794409472, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.25.post_attention_norm.weight", "offset": 3832944640, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 3832958976, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 3832973312, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3832987648, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.25.attn_output.weight", "offset": 3838492672, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.25.attn_q.weight", "offset": 3849502720, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.25.attn_v.weight", "offset": 3860512768, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.26.attn_norm.weight", "offset": 3866017792, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3866032128, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.26.ffn_gate.weight", "offset": 3904567296, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.26.ffn_up.weight", "offset": 3943102464, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.26.post_attention_norm.weight", "offset": 3981637632, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 3981651968, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 3981666304, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3981680640, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.26.attn_output.weight", "offset": 3987185664, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.26.attn_q.weight", "offset": 3998195712, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.26.attn_v.weight", "offset": 4009205760, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.27.attn_norm.weight", "offset": 4014710784, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4014725120, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.27.ffn_gate.weight", "offset": 4053260288, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.27.ffn_up.weight", "offset": 4091795456, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.27.post_attention_norm.weight", "offset": 4130330624, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 4130344960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 4130359296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 4130373632, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.27.attn_output.weight", "offset": 4135878656, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.27.attn_q.weight", "offset": 4146888704, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.27.attn_v.weight", "offset": 4157898752, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.28.attn_norm.weight", "offset": 4163403776, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 4163418112, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.28.ffn_gate.weight", "offset": 4201953280, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.28.ffn_up.weight", "offset": 4240488448, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.28.post_attention_norm.weight", "offset": 4279023616, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 4279037952, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 4279052288, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 4279066624, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.28.attn_output.weight", "offset": 4284571648, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.28.attn_q.weight", "offset": 4295581696, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.28.attn_v.weight", "offset": 4306591744, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.29.attn_norm.weight", "offset": 4312096768, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 4312111104, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.29.ffn_gate.weight", "offset": 4350646272, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.29.ffn_up.weight", "offset": 4389181440, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.29.post_attention_norm.weight", "offset": 4427716608, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 4427730944, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 4427745280, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 4427759616, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.29.attn_output.weight", "offset": 4433264640, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.29.attn_q.weight", "offset": 4444274688, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.29.attn_v.weight", "offset": 4455284736, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.30.attn_norm.weight", "offset": 4460789760, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 4460804096, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.30.ffn_gate.weight", "offset": 4499339264, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.30.ffn_up.weight", "offset": 4537874432, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.30.post_attention_norm.weight", "offset": 4576409600, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 4576423936, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 4576438272, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 4576452608, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.30.attn_output.weight", "offset": 4581957632, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.30.attn_q.weight", "offset": 4592967680, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.30.attn_v.weight", "offset": 4603977728, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.31.attn_k.weight", "offset": 4609482752, "shape": [ 3584, 2048 ], "size": 5505024, "type": 7 }, { "name": "blk.31.ffn_down.weight", "offset": 4614987776, "shape": [ 14336, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.31.ffn_gate.weight", "offset": 4653522944, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.31.ffn_up.weight", "offset": 4692058112, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.31.post_attention_norm.weight", "offset": 4730593280, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 4730607616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 4730621952, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 4730636288, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 4730650624, "shape": [ 4096, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.31.attn_q.weight", "offset": 4741660672, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.31.attn_v.weight", "offset": 4752670720, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.32.ffn_gate.weight", "offset": 4758175744, "shape": [ 3584, 14336 ], "size": 5505024, "type": 7 }, { "name": "blk.32.ffn_up.weight", "offset": 4796710912, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.32.attn_k.weight", "offset": 4835246080, "shape": [ 3584, 2048 ], "size": 38535168, "type": 7 }, { "name": "blk.32.attn_output.weight", "offset": 4840751104, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.32.attn_q.weight", "offset": 4851761152, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.32.attn_v.weight", "offset": 4862771200, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.32.attn_norm.weight", "offset": 4868276224, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 4868290560, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.32.post_attention_norm.weight", "offset": 4906825728, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 4906840064, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 4906854400, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 4906868736, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 4906883072, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.33.ffn_gate.weight", "offset": 4945418240, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.33.ffn_up.weight", "offset": 4983953408, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.33.post_attention_norm.weight", "offset": 5022488576, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 5022502912, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 5022517248, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 5022531584, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.33.attn_output.weight", "offset": 5028036608, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.33.attn_q.weight", "offset": 5039046656, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.33.attn_v.weight", "offset": 5050056704, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.34.attn_norm.weight", "offset": 5055561728, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 5055576064, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.34.ffn_gate.weight", "offset": 5094111232, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.34.ffn_up.weight", "offset": 5132646400, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.34.post_attention_norm.weight", "offset": 5171181568, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 5171195904, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 5171210240, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 5171224576, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.34.attn_output.weight", "offset": 5176729600, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.34.attn_q.weight", "offset": 5187739648, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.34.attn_v.weight", "offset": 5198749696, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.35.attn_norm.weight", "offset": 5204254720, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 5204269056, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.35.ffn_gate.weight", "offset": 5242804224, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.35.ffn_up.weight", "offset": 5281339392, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.35.post_attention_norm.weight", "offset": 5319874560, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 5319888896, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 5319903232, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 5319917568, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.35.attn_output.weight", "offset": 5325422592, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.35.attn_q.weight", "offset": 5336432640, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.35.attn_v.weight", "offset": 5347442688, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.36.ffn_norm.weight", "offset": 5352947712, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 5352962048, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.36.ffn_gate.weight", "offset": 5391497216, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.36.ffn_up.weight", "offset": 5430032384, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.36.post_attention_norm.weight", "offset": 5468567552, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 5468581888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 5468596224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 5468610560, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.36.attn_output.weight", "offset": 5474115584, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.36.attn_q.weight", "offset": 5485125632, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.36.attn_v.weight", "offset": 5496135680, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.37.attn_norm.weight", "offset": 5501640704, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 5501655040, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.37.ffn_gate.weight", "offset": 5540190208, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.37.ffn_up.weight", "offset": 5578725376, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.37.post_attention_norm.weight", "offset": 5617260544, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 5617274880, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 5617289216, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 5617303552, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.37.attn_output.weight", "offset": 5622808576, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.37.attn_q.weight", "offset": 5633818624, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.37.attn_v.weight", "offset": 5644828672, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.38.attn_norm.weight", "offset": 5650333696, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 5650348032, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.38.ffn_gate.weight", "offset": 5688883200, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.38.ffn_up.weight", "offset": 5727418368, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.38.post_attention_norm.weight", "offset": 5765953536, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 5765967872, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 5765982208, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 5765996544, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.38.attn_output.weight", "offset": 5771501568, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.38.attn_q.weight", "offset": 5782511616, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.38.attn_v.weight", "offset": 5793521664, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.39.attn_norm.weight", "offset": 5799026688, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 5799041024, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.39.ffn_gate.weight", "offset": 5837576192, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.39.ffn_up.weight", "offset": 5876111360, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.39.post_attention_norm.weight", "offset": 5914646528, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 5914660864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 5914675200, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 5914689536, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.39.attn_output.weight", "offset": 5920194560, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.39.attn_q.weight", "offset": 5931204608, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.39.attn_v.weight", "offset": 5942214656, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.40.attn_norm.weight", "offset": 5947719680, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 5947734016, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.40.ffn_gate.weight", "offset": 5986269184, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.40.ffn_up.weight", "offset": 6024804352, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.40.post_attention_norm.weight", "offset": 6063339520, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 6063353856, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 6063368192, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 6063382528, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.40.attn_output.weight", "offset": 6068887552, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.40.attn_q.weight", "offset": 6079897600, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.40.attn_v.weight", "offset": 6090907648, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "blk.41.attn_norm.weight", "offset": 6096412672, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 6096427008, "shape": [ 14336, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.41.ffn_gate.weight", "offset": 6134962176, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.41.ffn_up.weight", "offset": 6173497344, "shape": [ 3584, 14336 ], "size": 38535168, "type": 7 }, { "name": "blk.41.post_attention_norm.weight", "offset": 6212032512, "shape": [ 3584 ], "size": 38535168, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 6212046848, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 6212061184, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 6212075520, "shape": [ 3584, 2048 ], "size": 14336, "type": 7 }, { "name": "blk.41.attn_output.weight", "offset": 6217580544, "shape": [ 4096, 3584 ], "size": 5505024, "type": 7 }, { "name": "blk.41.attn_q.weight", "offset": 6228590592, "shape": [ 3584, 4096 ], "size": 11010048, "type": 7 }, { "name": "blk.41.attn_v.weight", "offset": 6239600640, "shape": [ 3584, 2048 ], "size": 11010048, "type": 7 }, { "name": "output_norm.weight", "offset": 6245105664, "shape": [ 3584 ], "size": 5505024, "type": 0 }, { "name": "token_embd.weight", "offset": 6245120000, "shape": [ 3584, 256000 ], "size": 14336, "type": 14 } ], "version": 3 }