gemma2

{ "metadata": { "gemma2.attention.head_count": 16, "gemma2.attention.head_count_kv": 8, "gemma2.attention.key_length": 256, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 256, "gemma2.block_count": 42, "gemma2.context_length": 8192, "gemma2.embedding_length": 3584, "gemma2.feed_forward_length": 14336, "general.architecture": "gemma2", "general.file_type": 12, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 9241705984, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 14336, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 35338240, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 57415680, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.0.post_attention_norm.weight", "offset": 79493120, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 79507456, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 79521792, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 79536128, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 82690048, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 90947584, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 97255424, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 102301696, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 102316032, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 137639936, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 159717376, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.1.post_attention_norm.weight", "offset": 181794816, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 181809152, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 181823488, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 181837824, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 184991744, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 193249280, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 199557120, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.2.attn_v.weight", "offset": 204603392, "shape": [ 3584, 2048 ], "size": 5046272, "type": 12 }, { "name": "blk.2.ffn_down.weight", "offset": 208732160, "shape": [ 14336, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.2.ffn_gate.weight", "offset": 237633536, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 259710976, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.2.post_attention_norm.weight", "offset": 281788416, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 281802752, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 281817088, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 281831424, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 284985344, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 293242880, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 299550720, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 299565056, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.3.ffn_down.weight", "offset": 303693824, "shape": [ 14336, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 332595200, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 354672640, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.3.post_attention_norm.weight", "offset": 376750080, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 376764416, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 376778752, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 376793088, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 379947008, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 388204544, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 394512384, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 394526720, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 394541056, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 423442432, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 445519872, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.4.post_attention_norm.weight", "offset": 467597312, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 467611648, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 467625984, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 467640320, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 470794240, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 479051776, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 485359616, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 489488384, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 489502720, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 518404096, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 540481536, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000blk.17.attn_v.weight\u0002\u0000\u0000\u0000\u0000\u000e", "offset": 562558976, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 562573312, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 562587648, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 562601984, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 565755904, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 574013440, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 580321280, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 584450048, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 584464384, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 613365760, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 635443200, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.6.post_attention_norm.weight", "offset": 657520640, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 657534976, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 657549312, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 657563648, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 660717568, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 668975104, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 675282944, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 679411712, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 679426048, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.7.attn_k.weight", "offset": 701503488, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 704657408, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 712914944, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 719222784, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 723351552, "shape": [ 3584, 14336 ], "size": 4128768, "type": 11 }, { "name": "blk.7.ffn_down.weight", "offset": 745428992, "shape": [ 14336, 3584 ], "size": 22077440, "type": 12 }, { "name": "blk.7.post_attention_norm.weight", "offset": 774330368, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 774344704, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 774359040, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 774373376, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 774387712, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 803289088, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 825366528, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 847443968, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 847458304, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 847472640, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 850626560, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 858884096, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 865191936, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.8.post_attention_norm.weight", "offset": 869320704, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 869335040, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.9.ffn_down.weight", "offset": 873463808, "shape": [ 14336, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 902365184, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.9.ffn_gate.weight", "offset": 902379520, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 924456960, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.9.post_attention_norm.weight", "offset": 946534400, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 946548736, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 946563072, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 946577408, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 949731328, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 957988864, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.10.attn_q.weight", "offset": 964296704, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 970604544, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.10.ffn_down.weight", "offset": 974733312, "shape": [ 14336, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.10.ffn_gate.weight", "offset": 1003634688, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.10.attn_k.weight", "offset": 1025712128, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 1028866048, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 1028880384, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.10.post_attention_norm.weight", "offset": 1050957824, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 1050972160, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1050986496, "shape": [ 4096, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1059244032, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1059258368, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.post_attention_norm.weight", "offset": 1059272704, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1059287040, "shape": [ 3584, 4096 ], "size": 14336, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1065594880, "shape": [ 3584, 14336 ], "size": 6307840, "type": 11 }, { "name": "blk.11.attn_k.weight", "offset": 1087672320, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 1090826240, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 1090840576, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1090854912, "shape": [ 4096, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.11.attn_v.weight", "offset": 1099112448, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.11.ffn_down.weight", "offset": 1103241216, "shape": [ 14336, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 1132142592, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 1154220032, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1154234368, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.12.attn_v.weight", "offset": 1183135744, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.12.attn_q.weight", "offset": 1187264512, "shape": [ 3584, 4096 ], "size": 4128768, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 1193572352, "shape": [ 4096, 3584 ], "size": 6307840, "type": 12 }, { "name": "blk.12.attn_k.weight", "offset": 1201829888, "shape": [ 3584, 2048 ], "size": 8257536, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 1204983808, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 1204998144, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.post_attention_norm.weight", "offset": 1205012480, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 1205026816, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 1227104256, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 1249181696, "shape": [ 4096, 3584 ], "size": 22077440, "type": 12 }, { "name": "blk.13.attn_v.weight", "offset": 1257439232, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 1261568000, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 1261582336, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.post_attention_norm.weight", "offset": 1261596672, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 1261611008, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 1283688448, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.13.ffn_down.weight", "offset": 1305765888, "shape": [ 14336, 3584 ], "size": 22077440, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 1334667264, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1334681600, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.13.attn_q.weight", "offset": 1337835520, "shape": [ 3584, 4096 ], "size": 3153920, "type": 11 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 1344143360, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 1344157696, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1344172032, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 1373073408, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 1395150848, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.14.post_attention_norm.weight", "offset": 1417228288, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 1417242624, "shape": [ 4096, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 1425500160, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1425514496, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 1428668416, "shape": [ 3584, 2048 ], "size": 3153920, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 1432797184, "shape": [ 3584, 4096 ], "size": 4128768, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 1439105024, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 1443233792, "shape": [ 3584, 4096 ], "size": 4128768, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 1449541632, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1449555968, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 1478457344, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 1500534784, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.15.post_attention_norm.weight", "offset": 1522612224, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 1522626560, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 1522640896, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1522655232, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 1525809152, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 1534066688, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 1534081024, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1534095360, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 1537249280, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1537263616, "shape": [ 3584, 4096 ], "size": 14336, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 1543571456, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 1547700224, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.16.post_attention_norm.weight", "offset": 1555957760, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 1555972096, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 1578049536, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.16.ffn_down.weight", "offset": 1600126976, "shape": [ 14336, 3584 ], "size": 22077440, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 1629028352, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 1629042688, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 1629057024, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 1632210944, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 1640468480, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.17.post_attention_norm.weight", "offset": 1640482816, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 1640497152, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 1662574592, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.17.ffn_down.weight", "offset": 1684652032, "shape": [ 14336, 3584 ], "size": 22077440, "type": 12 }, { "name": "ffn_gate.weight\u0002\u0000\u0000\u0000\u0000", "offset": 1713553408, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 1717682176, "shape": [ 3584, 4096 ], "size": 4128768, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 1723990016, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 1728118784, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 1728133120, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 1728147456, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 1757048832, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 1779126272, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.18.post_attention_norm.weight", "offset": 1801203712, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 1801218048, "shape": [ 3584, 4096 ], "size": 14336, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 1807525888, "shape": [ 4096, 3584 ], "size": 6307840, "type": 12 }, { "name": "blk.18.attn_k.weight", "offset": 1815783424, "shape": [ 3584, 2048 ], "size": 8257536, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 1818937344, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 1818951680, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 1818966016, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 1818980352, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 1847881728, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 1869959168, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.19.post_attention_norm.weight", "offset": 1892036608, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 1892050944, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 1892065280, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 1895219200, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 1903476736, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 1909784576, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.20.attn_k.weight", "offset": 1913913344, "shape": [ 3584, 2048 ], "size": 4128768, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 1917067264, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 1925324800, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 1931632640, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 1935761408, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 1935775744, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 1964677120, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 1986754560, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.20.post_attention_norm.weight", "offset": 2008832000, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 2008846336, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 2008860672, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2008875008, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 2037776384, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.21.ffn_gate.weight", "offset": 2037790720, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 2059868160, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.21.post_attention_norm.weight", "offset": 2081945600, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 2081959936, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 2081974272, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2081988608, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 2085142528, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 2093400064, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 2099707904, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 2103836672, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2103851008, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 2132752384, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 2154829824, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.22.post_attention_norm.weight", "offset": 2176907264, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 2176921600, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 2176935936, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2176950272, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 2180104192, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 2188361728, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 2194669568, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 2198798336, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2198812672, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 2227714048, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 2249791488, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.23.post_attention_norm.weight", "offset": 2271868928, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 2271883264, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 2271897600, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2271911936, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 2275065856, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 2283323392, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 2289631232, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 2293760000, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2293774336, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 2322675712, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 2344753152, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.24.post_attention_norm.weight", "offset": 2366830592, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 2366844928, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 2366859264, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 2366873600, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 2370027520, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 2378285056, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 2384592896, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 2388721664, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 2388736000, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 2417637376, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 2439714816, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.25.post_attention_norm.weight", "offset": 2461792256, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 2461806592, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 2461820928, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 2461835264, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 2464989184, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 2473246720, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 2479554560, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 2483683328, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 2483697664, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 2512599040, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 2534676480, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.26.post_attention_norm.weight", "offset": 2556753920, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 2556768256, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 2556782592, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 2556796928, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 2559950848, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 2568208384, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 2574516224, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 2578644992, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 2578659328, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 2607560704, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 2629638144, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.27.post_attention_norm.weight", "offset": 2651715584, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 2651729920, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 2651744256, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 2651758592, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 2654912512, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.27.attn_q.weight", "offset": 2663170048, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 2669477888, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 2673606656, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 2673620992, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.28.ffn_gate.weight", "offset": 2702522368, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 2724599808, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.28.post_attention_norm.weight", "offset": 2746677248, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 2746691584, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 2746705920, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 2746720256, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 2749874176, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.28.attn_q.weight", "offset": 2758131712, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 2764439552, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 2768568320, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 2768582656, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.29.ffn_gate.weight", "offset": 2797484032, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 2819561472, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.29.post_attention_norm.weight", "offset": 2841638912, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 2841653248, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 2841667584, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 2841681920, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 2844835840, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.29.attn_q.weight", "offset": 2853093376, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 2859401216, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 2863529984, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 2863544320, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.30.ffn_gate.weight", "offset": 2892445696, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 2914523136, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.30.post_attention_norm.weight", "offset": 2936600576, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 2936614912, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 2936629248, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 2936643584, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 2939797504, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.30.attn_q.weight", "offset": 2948055040, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 2954362880, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.31.attn_k.weight", "offset": 2958491648, "shape": [ 3584, 2048 ], "size": 4128768, "type": 11 }, { "name": "blk.31.ffn_down.weight", "offset": 2961645568, "shape": [ 14336, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.31.ffn_gate.weight", "offset": 2990546944, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 3012624384, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.31.post_attention_norm.weight", "offset": 3034701824, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 3034716160, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 3034730496, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 3034744832, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 3034759168, "shape": [ 4096, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.31.attn_q.weight", "offset": 3043016704, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 3049324544, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.32.ffn_gate.weight", "offset": 3053453312, "shape": [ 3584, 14336 ], "size": 4128768, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 3075530752, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.32.attn_k.weight", "offset": 3097608192, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 3100762112, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.32.attn_q.weight", "offset": 3109019648, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.32.attn_v.weight", "offset": 3115327488, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 3119456256, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 3119470592, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.32.post_attention_norm.weight", "offset": 3148371968, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 3148386304, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 3148400640, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 3148414976, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 3148429312, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.33.ffn_gate.weight", "offset": 3177330688, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 3199408128, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.33.post_attention_norm.weight", "offset": 3221485568, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 3221499904, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 3221514240, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 3221528576, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 3224682496, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.33.attn_q.weight", "offset": 3232940032, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.33.attn_v.weight", "offset": 3239247872, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 3243376640, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 3243390976, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.34.ffn_gate.weight", "offset": 3272292352, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 3294369792, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.34.post_attention_norm.weight", "offset": 3316447232, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 3316461568, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 3316475904, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 3316490240, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 3319644160, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.34.attn_q.weight", "offset": 3327901696, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.34.attn_v.weight", "offset": 3334209536, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.35.attn_norm.weight", "offset": 3338338304, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 3338352640, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.35.ffn_gate.weight", "offset": 3367254016, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 3389331456, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.35.post_attention_norm.weight", "offset": 3411408896, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 3411423232, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 3411437568, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 3411451904, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 3414605824, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.35.attn_q.weight", "offset": 3422863360, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.35.attn_v.weight", "offset": 3429171200, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.36.ffn_norm.weight", "offset": 3433299968, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 3433314304, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.36.ffn_gate.weight", "offset": 3462215680, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 3484293120, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.36.post_attention_norm.weight", "offset": 3506370560, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 3506384896, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 3506399232, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 3506413568, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 3509567488, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.36.attn_q.weight", "offset": 3517825024, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.36.attn_v.weight", "offset": 3524132864, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 3528261632, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 3528275968, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.37.ffn_gate.weight", "offset": 3557177344, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 3579254784, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.37.post_attention_norm.weight", "offset": 3601332224, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 3601346560, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 3601360896, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 3601375232, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 3604529152, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.37.attn_q.weight", "offset": 3612786688, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.37.attn_v.weight", "offset": 3619094528, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 3623223296, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 3623237632, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.38.ffn_gate.weight", "offset": 3652139008, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 3674216448, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.38.post_attention_norm.weight", "offset": 3696293888, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 3696308224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 3696322560, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 3696336896, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 3699490816, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.38.attn_q.weight", "offset": 3707748352, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.38.attn_v.weight", "offset": 3714056192, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 3718184960, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 3718199296, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.39.ffn_gate.weight", "offset": 3747100672, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 3769178112, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.39.post_attention_norm.weight", "offset": 3791255552, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 3791269888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 3791284224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 3791298560, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 3794452480, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.39.attn_q.weight", "offset": 3802710016, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.39.attn_v.weight", "offset": 3809017856, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.40.attn_norm.weight", "offset": 3813146624, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 3813160960, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.40.ffn_gate.weight", "offset": 3842062336, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.40.ffn_up.weight", "offset": 3864139776, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.40.post_attention_norm.weight", "offset": 3886217216, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 3886231552, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 3886245888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 3886260224, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 3889414144, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.40.attn_q.weight", "offset": 3897671680, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.40.attn_v.weight", "offset": 3903979520, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "blk.41.attn_norm.weight", "offset": 3908108288, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 3908122624, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.41.ffn_gate.weight", "offset": 3937024000, "shape": [ 3584, 14336 ], "size": 28901376, "type": 11 }, { "name": "blk.41.ffn_up.weight", "offset": 3959101440, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.41.post_attention_norm.weight", "offset": 3981178880, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 3981193216, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 3981207552, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 3981221888, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 3984375808, "shape": [ 4096, 3584 ], "size": 3153920, "type": 12 }, { "name": "blk.41.attn_q.weight", "offset": 3992633344, "shape": [ 3584, 4096 ], "size": 8257536, "type": 11 }, { "name": "blk.41.attn_v.weight", "offset": 3998941184, "shape": [ 3584, 2048 ], "size": 6307840, "type": 12 }, { "name": "output_norm.weight", "offset": 4003069952, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "token_embd.weight", "offset": 4003084288, "shape": [ 3584, 256000 ], "size": 14336, "type": 14 } ], "version": 3 }