gemma2

{ "metadata": { "gemma2.attention.head_count": 32, "gemma2.attention.head_count_kv": 16, "gemma2.attention.key_length": 128, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 128, "gemma2.block_count": 46, "gemma2.context_length": 8192, "gemma2.embedding_length": 4608, "gemma2.feed_forward_length": 36864, "general.architecture": "gemma2", "general.file_type": 18, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 27227128320, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 18432, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 139364352, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 278710272, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.0.post_attention_norm.weight", "offset": 418056192, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 418074624, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 418093056, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 418111488, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.0.attn_output.weight", "offset": 425852928, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.0.attn_q.weight", "offset": 441335808, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.0.attn_v.weight", "offset": 456818688, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 464560128, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 464578560, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 603924480, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 743270400, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.1.post_attention_norm.weight", "offset": 882616320, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 882634752, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 882653184, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 882671616, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.1.attn_output.weight", "offset": 890413056, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.1.attn_q.weight", "offset": 905895936, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.1.attn_v.weight", "offset": 921378816, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.2.attn_k.weight", "offset": 929120256, "shape": [ 4608, 2048 ], "size": 7741440, "type": 14 }, { "name": "blk.2.attn_output.weight", "offset": 936861696, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.2.attn_q.weight", "offset": 952344576, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.2.attn_v.weight", "offset": 967827456, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 975568896, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 975587328, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 1114933248, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 1254279168, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.2.post_attention_norm.weight", "offset": 1393625088, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 1393643520, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 1393661952, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_norm.weight", "offset": 1393680384, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1393698816, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 1533044736, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.3.ffn_up.weight", "offset": 1672390656, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.3.post_attention_norm.weight", "offset": 1811736576, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 1811755008, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 1811773440, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1811791872, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.3.attn_output.weight", "offset": 1819533312, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.3.attn_q.weight", "offset": 1835016192, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.3.attn_v.weight", "offset": 1850499072, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1858240512, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1858258944, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 1997604864, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 2136950784, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.4.post_attention_norm.weight", "offset": 2276296704, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 2276315136, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 2276333568, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 2276352000, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.4.attn_output.weight", "offset": 2284093440, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.4.attn_q.weight", "offset": 2299576320, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.4.attn_v.weight", "offset": 2315059200, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 2322800640, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 2322819072, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.5.ffn_gate.weight", "offset": 2462164992, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.5.ffn_up.weight", "offset": 2601510912, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.5.post_attention_norm.weight", "offset": 2740856832, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 2740875264, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 2740893696, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 2740912128, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.5.attn_output.weight", "offset": 2748653568, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.5.attn_q.weight", "offset": 2764136448, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.5.attn_v.weight", "offset": 2779619328, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.6.ffn_gate.weight", "offset": 2787360768, "shape": [ 4608, 36864 ], "size": 7741440, "type": 14 }, { "name": "blk.6.attn_k.weight", "offset": 2926706688, "shape": [ 4608, 2048 ], "size": 139345920, "type": 14 }, { "name": "blk.6.attn_output.weight", "offset": 2934448128, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.6.attn_q.weight", "offset": 2949931008, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.6.attn_v.weight", "offset": 2965413888, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 2973155328, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2973173760, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.6.ffn_up.weight", "offset": 3112519680, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.6.post_attention_norm.weight", "offset": 3251865600, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 3251884032, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 3251902464, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 3251920896, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 3251939328, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 3391285248, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.7.ffn_up.weight", "offset": 3530631168, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.7.post_attention_norm.weight", "offset": 3669977088, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 3669995520, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 3670013952, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 3670032384, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.7.attn_output.weight", "offset": 3677773824, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.7.attn_q.weight", "offset": 3693256704, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.7.attn_v.weight", "offset": 3708739584, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.8.attn_norm.weight", "offset": 3716481024, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3716499456, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.8.ffn_gate.weight", "offset": 3855845376, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.8.ffn_up.weight", "offset": 3995191296, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.8.post_attention_norm.weight", "offset": 4134537216, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 4134555648, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 4134574080, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 4134592512, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.8.attn_output.weight", "offset": 4142333952, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.8.attn_q.weight", "offset": 4157816832, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.8.attn_v.weight", "offset": 4173299712, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 4181041152, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 4181059584, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 4320405504, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.9.ffn_up.weight", "offset": 4459751424, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.9.post_attention_norm.weight", "offset": 4599097344, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 4599115776, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 4599134208, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 4599152640, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.9.attn_output.weight", "offset": 4606894080, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.9.attn_q.weight", "offset": 4622376960, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.9.attn_v.weight", "offset": 4637859840, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 4645601280, "shape": [ 4608, 36864 ], "size": 7741440, "type": 14 }, { "name": "blk.10.ffn_up.weight", "offset": 4784947200, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.10.attn_k.weight", "offset": 4924293120, "shape": [ 4608, 2048 ], "size": 139345920, "type": 14 }, { "name": "blk.10.attn_output.weight", "offset": 4932034560, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.10.attn_q.weight", "offset": 4947517440, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.10.attn_v.weight", "offset": 4963000320, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 4970741760, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 4970760192, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.10.post_attention_norm.weight", "offset": 5110106112, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 5110124544, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 5110142976, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_norm.weight", "offset": 5110161408, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 5110179840, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.11.ffn_gate.weight", "offset": 5249525760, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.11.ffn_up.weight", "offset": 5388871680, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.11.post_attention_norm.weight", "offset": 5528217600, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 5528236032, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 5528254464, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 5528272896, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.11.attn_output.weight", "offset": 5536014336, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.11.attn_q.weight", "offset": 5551497216, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.11.attn_v.weight", "offset": 5566980096, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 5574721536, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 5574739968, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.12.ffn_gate.weight", "offset": 5714085888, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.12.ffn_up.weight", "offset": 5853431808, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.12.post_attention_norm.weight", "offset": 5992777728, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 5992796160, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 5992814592, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 5992833024, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.12.attn_output.weight", "offset": 6000574464, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.12.attn_q.weight", "offset": 6016057344, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.12.attn_v.weight", "offset": 6031540224, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 6039281664, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 6039300096, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.13.ffn_gate.weight", "offset": 6178646016, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.13.ffn_up.weight", "offset": 6317991936, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.13.post_attention_norm.weight", "offset": 6457337856, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 6457356288, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 6457374720, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 6457393152, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.13.attn_output.weight", "offset": 6465134592, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.13.attn_q.weight", "offset": 6480617472, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.13.attn_v.weight", "offset": 6496100352, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 6503841792, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 6503860224, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.14.ffn_gate.weight", "offset": 6643206144, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.14.ffn_up.weight", "offset": 6782552064, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.14.post_attention_norm.weight", "offset": 6921897984, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 6921916416, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 6921934848, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 6921953280, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.14.attn_output.weight", "offset": 6929694720, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.14.attn_q.weight", "offset": 6945177600, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.14.attn_v.weight", "offset": 6960660480, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.15.attn_k.weight", "offset": 6968401920, "shape": [ 4608, 2048 ], "size": 7741440, "type": 14 }, { "name": "blk.15.attn_output.weight", "offset": 6976143360, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.15.attn_q.weight", "offset": 6991626240, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.15.attn_v.weight", "offset": 7007109120, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.15.attn_norm.weight", "offset": 7014850560, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 7014868992, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.15.ffn_gate.weight", "offset": 7154214912, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.15.ffn_up.weight", "offset": 7293560832, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.15.post_attention_norm.weight", "offset": 7432906752, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 7432925184, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 7432943616, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 7432962048, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 7432980480, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 7572326400, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.16.ffn_up.weight", "offset": 7711672320, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.16.post_attention_norm.weight", "offset": 7851018240, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 7851036672, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 7851055104, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 7851073536, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.16.attn_output.weight", "offset": 7858814976, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.16.attn_q.weight", "offset": 7874297856, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.16.attn_v.weight", "offset": 7889780736, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 7897522176, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 7897540608, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.17.ffn_gate.weight", "offset": 8036886528, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.17.ffn_up.weight", "offset": 8176232448, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.17.post_attention_norm.weight", "offset": 8315578368, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 8315596800, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 8315615232, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 8315633664, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.17.attn_output.weight", "offset": 8323375104, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.17.attn_q.weight", "offset": 8338857984, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.17.attn_v.weight", "offset": 8354340864, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 8362082304, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 8362100736, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.18.ffn_gate.weight", "offset": 8501446656, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.18.ffn_up.weight", "offset": 8640792576, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.18.post_attention_norm.weight", "offset": 8780138496, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 8780156928, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 8780175360, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 8780193792, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.18.attn_output.weight", "offset": 8787935232, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.18.attn_q.weight", "offset": 8803418112, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.18.attn_v.weight", "offset": 8818900992, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.19.ffn_gate.weight", "offset": 8826642432, "shape": [ 4608, 36864 ], "size": 7741440, "type": 14 }, { "name": "blk.19.attn_k.weight", "offset": 8965988352, "shape": [ 4608, 2048 ], "size": 139345920, "type": 14 }, { "name": "blk.19.attn_output.weight", "offset": 8973729792, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.19.attn_q.weight", "offset": 8989212672, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.19.attn_v.weight", "offset": 9004695552, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 9012436992, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 9012455424, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 9151801344, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.19.post_attention_norm.weight", "offset": 9291147264, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 9291165696, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 9291184128, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_norm.weight", "offset": 9291202560, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 9291220992, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.20.ffn_gate.weight", "offset": 9430566912, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.20.ffn_up.weight", "offset": 9569912832, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.20.post_attention_norm.weight", "offset": 9709258752, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 9709277184, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 9709295616, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 9709314048, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.20.attn_output.weight", "offset": 9717055488, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.20.attn_q.weight", "offset": 9732538368, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.20.attn_v.weight", "offset": 9748021248, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 9755762688, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 9755781120, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.21.ffn_gate.weight", "offset": 9895127040, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.21.ffn_up.weight", "offset": 10034472960, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.21.post_attention_norm.weight", "offset": 10173818880, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 10173837312, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 10173855744, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 10173874176, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.21.attn_output.weight", "offset": 10181615616, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.21.attn_q.weight", "offset": 10197098496, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.21.attn_v.weight", "offset": 10212581376, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 10220322816, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 10220341248, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.22.ffn_gate.weight", "offset": 10359687168, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.22.ffn_up.weight", "offset": 10499033088, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.22.post_attention_norm.weight", "offset": 10638379008, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 10638397440, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 10638415872, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 10638434304, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.22.attn_output.weight", "offset": 10646175744, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.22.attn_q.weight", "offset": 10661658624, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.22.attn_v.weight", "offset": 10677141504, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.23.ffn_up.weight", "offset": 10684882944, "shape": [ 4608, 36864 ], "size": 7741440, "type": 14 }, { "name": "blk.23.ffn_gate.weight", "offset": 10824228864, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.23.attn_k.weight", "offset": 10963574784, "shape": [ 4608, 2048 ], "size": 139345920, "type": 14 }, { "name": "blk.23.attn_output.weight", "offset": 10971316224, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.23.attn_q.weight", "offset": 10986799104, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.23.attn_v.weight", "offset": 11002281984, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.23.attn_norm.weight", "offset": 11010023424, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 11010041856, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.23.post_attention_norm.weight", "offset": 11149387776, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 11149406208, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 11149424640, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 11149443072, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 11149461504, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.24.ffn_gate.weight", "offset": 11288807424, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.24.ffn_up.weight", "offset": 11428153344, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.24.post_attention_norm.weight", "offset": 11567499264, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 11567517696, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 11567536128, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 11567554560, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.24.attn_output.weight", "offset": 11575296000, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.24.attn_q.weight", "offset": 11590778880, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.24.attn_v.weight", "offset": 11606261760, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 11614003200, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 11614021632, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 11753367552, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.25.ffn_up.weight", "offset": 11892713472, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.25.post_attention_norm.weight", "offset": 12032059392, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 12032077824, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 12032096256, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 12032114688, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.25.attn_output.weight", "offset": 12039856128, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.25.attn_q.weight", "offset": 12055339008, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.25.attn_v.weight", "offset": 12070821888, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 12078563328, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 12078581760, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.26.ffn_gate.weight", "offset": 12217927680, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.26.ffn_up.weight", "offset": 12357273600, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.26.post_attention_norm.weight", "offset": 12496619520, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 12496637952, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 12496656384, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 12496674816, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.26.attn_output.weight", "offset": 12504416256, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.26.attn_q.weight", "offset": 12519899136, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.26.attn_v.weight", "offset": 12535382016, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 12543123456, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 12543141888, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 12682487808, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.27.ffn_up.weight", "offset": 12821833728, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.27.post_attention_norm.weight", "offset": 12961179648, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 12961198080, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 12961216512, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 12961234944, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.27.attn_output.weight", "offset": 12968976384, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.27.attn_q.weight", "offset": 12984459264, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.27.attn_v.weight", "offset": 12999942144, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.28.attn_k.weight", "offset": 13007683584, "shape": [ 4608, 2048 ], "size": 7741440, "type": 14 }, { "name": "blk.28.attn_output.weight", "offset": 13015425024, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.28.attn_q.weight", "offset": 13030907904, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.28.attn_v.weight", "offset": 13046390784, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.28.attn_norm.weight", "offset": 13054132224, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 13054150656, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.28.ffn_gate.weight", "offset": 13193496576, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.28.ffn_up.weight", "offset": 13332842496, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.28.post_attention_norm.weight", "offset": 13472188416, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 13472206848, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 13472225280, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 13472243712, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 13472262144, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.29.ffn_gate.weight", "offset": 13611608064, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.29.ffn_up.weight", "offset": 13750953984, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.29.post_attention_norm.weight", "offset": 13890299904, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 13890318336, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 13890336768, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 13890355200, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.29.attn_output.weight", "offset": 13898096640, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.29.attn_q.weight", "offset": 13913579520, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.29.attn_v.weight", "offset": 13929062400, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.30.attn_norm.weight", "offset": 13936803840, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 13936822272, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.30.ffn_gate.weight", "offset": 14076168192, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.30.ffn_up.weight", "offset": 14215514112, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.30.post_attention_norm.weight", "offset": 14354860032, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 14354878464, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 14354896896, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 14354915328, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.30.attn_output.weight", "offset": 14362656768, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.30.attn_q.weight", "offset": 14378139648, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.30.attn_v.weight", "offset": 14393622528, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.31.attn_norm.weight", "offset": 14401363968, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 14401382400, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.31.ffn_gate.weight", "offset": 14540728320, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.31.ffn_up.weight", "offset": 14680074240, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.31.post_attention_norm.weight", "offset": 14819420160, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 14819438592, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 14819457024, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 14819475456, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.31.attn_output.weight", "offset": 14827216896, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.31.attn_q.weight", "offset": 14842699776, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.31.attn_v.weight", "offset": 14858182656, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.32.ffn_gate.weight", "offset": 14865924096, "shape": [ 4608, 36864 ], "size": 7741440, "type": 14 }, { "name": "blk.32.attn_k.weight", "offset": 15005270016, "shape": [ 4608, 2048 ], "size": 139345920, "type": 14 }, { "name": "blk.32.attn_output.weight", "offset": 15013011456, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.32.attn_q.weight", "offset": 15028494336, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.32.attn_v.weight", "offset": 15043977216, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.32.attn_norm.weight", "offset": 15051718656, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 15051737088, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.32.ffn_up.weight", "offset": 15191083008, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.32.post_attention_norm.weight", "offset": 15330428928, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 15330447360, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 15330465792, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 15330484224, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 15330502656, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.33.ffn_gate.weight", "offset": 15469848576, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.33.ffn_up.weight", "offset": 15609194496, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.33.post_attention_norm.weight", "offset": 15748540416, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 15748558848, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 15748577280, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 15748595712, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.33.attn_output.weight", "offset": 15756337152, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.33.attn_q.weight", "offset": 15771820032, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.33.attn_v.weight", "offset": 15787302912, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.34.ffn_norm.weight", "offset": 15795044352, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 15795062784, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.34.ffn_gate.weight", "offset": 15934408704, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.34.ffn_up.weight", "offset": 16073754624, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.34.post_attention_norm.weight", "offset": 16213100544, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 16213118976, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_norm.weight", "offset": 16213137408, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 16213155840, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.34.attn_output.weight", "offset": 16220897280, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.34.attn_q.weight", "offset": 16236380160, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.34.attn_v.weight", "offset": 16251863040, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 16259604480, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 16259622912, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.35.ffn_gate.weight", "offset": 16398968832, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.35.ffn_up.weight", "offset": 16538314752, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.35.post_attention_norm.weight", "offset": 16677660672, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 16677679104, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 16677697536, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 16677715968, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.35.attn_output.weight", "offset": 16685457408, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.35.attn_q.weight", "offset": 16700940288, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.35.attn_v.weight", "offset": 16716423168, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.36.ffn_gate.weight", "offset": 16724164608, "shape": [ 4608, 36864 ], "size": 7741440, "type": 14 }, { "name": "blk.36.ffn_up.weight", "offset": 16863510528, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.36.attn_k.weight", "offset": 17002856448, "shape": [ 4608, 2048 ], "size": 139345920, "type": 14 }, { "name": "blk.36.attn_output.weight", "offset": 17010597888, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.36.attn_q.weight", "offset": 17026080768, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.36.attn_v.weight", "offset": 17041563648, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.36.attn_norm.weight", "offset": 17049305088, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 17049323520, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.36.post_attention_norm.weight", "offset": 17188669440, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 17188687872, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 17188706304, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 17188724736, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 17188743168, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.37.ffn_gate.weight", "offset": 17328089088, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.37.ffn_up.weight", "offset": 17467435008, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.37.post_attention_norm.weight", "offset": 17606780928, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 17606799360, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 17606817792, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 17606836224, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.37.attn_output.weight", "offset": 17614577664, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.37.attn_q.weight", "offset": 17630060544, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.37.attn_v.weight", "offset": 17645543424, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 17653284864, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 17653303296, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.38.ffn_gate.weight", "offset": 17792649216, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.38.ffn_up.weight", "offset": 17931995136, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.38.post_attention_norm.weight", "offset": 18071341056, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 18071359488, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 18071377920, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 18071396352, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.38.attn_output.weight", "offset": 18079137792, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.38.attn_q.weight", "offset": 18094620672, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.38.attn_v.weight", "offset": 18110103552, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.39.attn_norm.weight", "offset": 18117844992, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 18117863424, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.39.ffn_gate.weight", "offset": 18257209344, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.39.ffn_up.weight", "offset": 18396555264, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.39.post_attention_norm.weight", "offset": 18535901184, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 18535919616, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 18535938048, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 18535956480, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.39.attn_output.weight", "offset": 18543697920, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.39.attn_q.weight", "offset": 18559180800, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.39.attn_v.weight", "offset": 18574663680, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.40.ffn_down.weight", "offset": 18582405120, "shape": [ 36864, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.40.attn_norm.weight", "offset": 18721751040, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.40.ffn_gate.weight", "offset": 18721769472, "shape": [ 4608, 36864 ], "size": 18432, "type": 14 }, { "name": "blk.40.ffn_up.weight", "offset": 18861115392, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.40.post_attention_norm.weight", "offset": 19000461312, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 19000479744, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 19000498176, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 19000516608, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.40.attn_output.weight", "offset": 19008258048, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.40.attn_q.weight", "offset": 19023740928, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.40.attn_v.weight", "offset": 19039223808, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.41.attn_k.weight", "offset": 19046965248, "shape": [ 4608, 2048 ], "size": 7741440, "type": 14 }, { "name": "blk.41.attn_output.weight", "offset": 19054706688, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.41.attn_q.weight", "offset": 19070189568, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.41.attn_v.weight", "offset": 19085672448, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 19093413888, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 19093432320, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.41.ffn_gate.weight", "offset": 19232778240, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.41.ffn_up.weight", "offset": 19372124160, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.41.post_attention_norm.weight", "offset": 19511470080, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 19511488512, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 19511506944, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 19511525376, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 19511543808, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.42.ffn_gate.weight", "offset": 19650889728, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.42.ffn_up.weight", "offset": 19790235648, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.42.post_attention_norm.weight", "offset": 19929581568, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.42.post_ffw_norm.weight", "offset": 19929600000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 19929618432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 19929636864, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.42.attn_output.weight", "offset": 19937378304, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.42.attn_q.weight", "offset": 19952861184, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.42.attn_v.weight", "offset": 19968344064, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.43.attn_norm.weight", "offset": 19976085504, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 19976103936, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.43.ffn_gate.weight", "offset": 20115449856, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.43.ffn_up.weight", "offset": 20254795776, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.43.post_attention_norm.weight", "offset": 20394141696, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.43.post_ffw_norm.weight", "offset": 20394160128, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 20394178560, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 20394196992, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.43.attn_output.weight", "offset": 20401938432, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.43.attn_q.weight", "offset": 20417421312, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.43.attn_v.weight", "offset": 20432904192, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.44.attn_norm.weight", "offset": 20440645632, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 20440664064, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.44.ffn_gate.weight", "offset": 20580009984, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.44.ffn_up.weight", "offset": 20719355904, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.44.post_attention_norm.weight", "offset": 20858701824, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.44.post_ffw_norm.weight", "offset": 20858720256, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 20858738688, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 20858757120, "shape": [ 4608, 2048 ], "size": 18432, "type": 14 }, { "name": "blk.44.attn_output.weight", "offset": 20866498560, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.44.attn_q.weight", "offset": 20881981440, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.44.attn_v.weight", "offset": 20897464320, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.45.ffn_gate.weight", "offset": 20905205760, "shape": [ 4608, 36864 ], "size": 7741440, "type": 14 }, { "name": "blk.45.attn_k.weight", "offset": 21044551680, "shape": [ 4608, 2048 ], "size": 139345920, "type": 14 }, { "name": "blk.45.attn_output.weight", "offset": 21052293120, "shape": [ 4096, 4608 ], "size": 7741440, "type": 14 }, { "name": "blk.45.attn_q.weight", "offset": 21067776000, "shape": [ 4608, 4096 ], "size": 15482880, "type": 14 }, { "name": "blk.45.attn_v.weight", "offset": 21083258880, "shape": [ 4608, 2048 ], "size": 15482880, "type": 14 }, { "name": "blk.45.attn_norm.weight", "offset": 21091000320, "shape": [ 4608 ], "size": 7741440, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 21091018752, "shape": [ 36864, 4608 ], "size": 18432, "type": 14 }, { "name": "blk.45.ffn_up.weight", "offset": 21230364672, "shape": [ 4608, 36864 ], "size": 139345920, "type": 14 }, { "name": "blk.45.post_attention_norm.weight", "offset": 21369710592, "shape": [ 4608 ], "size": 139345920, "type": 0 }, { "name": "blk.45.post_ffw_norm.weight", "offset": 21369729024, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 21369747456, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "output_norm.weight", "offset": 21369765888, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "token_embd.weight", "offset": 21369784320, "shape": [ 4608, 256000 ], "size": 18432, "type": 14 } ], "version": 3 }