gemma2

{ "metadata": { "gemma2.attention.head_count": 16, "gemma2.attention.head_count_kv": 8, "gemma2.attention.key_length": 256, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 256, "gemma2.block_count": 42, "gemma2.context_length": 8192, "gemma2.embedding_length": 3584, "gemma2.feed_forward_length": 14336, "general.architecture": "gemma2", "general.file_type": 7, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 9241705984, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 14336, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.0.ffn_gate.weight", "offset": 54605824, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 109197312, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.0.post_attention_norm.weight", "offset": 163788800, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 163803136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 163817472, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 163831808, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 171630592, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.0.attn_q.weight", "offset": 187228160, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.0.attn_v.weight", "offset": 202825728, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 210624512, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 210638848, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.1.ffn_gate.weight", "offset": 265230336, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.1.ffn_up.weight", "offset": 319821824, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.1.post_attention_norm.weight", "offset": 374413312, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 374427648, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 374441984, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 374456320, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 382255104, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.1.attn_q.weight", "offset": 397852672, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.1.attn_v.weight", "offset": 413450240, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.2.attn_v.weight", "offset": 421249024, "shape": [ 3584, 2048 ], "size": 7798784, "type": 8 }, { "name": "blk.2.ffn_down.weight", "offset": 429047808, "shape": [ 14336, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.2.ffn_gate.weight", "offset": 483639296, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.2.ffn_up.weight", "offset": 538230784, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.2.post_attention_norm.weight", "offset": 592822272, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 592836608, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 592850944, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 592865280, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 600664064, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.2.attn_q.weight", "offset": 616261632, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 631859200, "shape": [ 3584 ], "size": 15597568, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 631873536, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.3.ffn_down.weight", "offset": 639672320, "shape": [ 14336, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.3.ffn_gate.weight", "offset": 694263808, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.3.ffn_up.weight", "offset": 748855296, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.3.post_attention_norm.weight", "offset": 803446784, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 803461120, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 803475456, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 803489792, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 811288576, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.3.attn_q.weight", "offset": 826886144, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.3.attn_norm.weight", "offset": 842483712, "shape": [ 3584 ], "size": 15597568, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 842498048, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 842512384, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.4.ffn_gate.weight", "offset": 897103872, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.4.ffn_up.weight", "offset": 951695360, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.4.post_attention_norm.weight", "offset": 1006286848, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 1006301184, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 1006315520, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1006329856, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 1014128640, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.4.attn_q.weight", "offset": 1029726208, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.4.attn_v.weight", "offset": 1045323776, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.5.attn_norm.weight", "offset": 1053122560, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1053136896, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.5.ffn_gate.weight", "offset": 1107728384, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.5.ffn_up.weight", "offset": 1162319872, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.5.post_attention_norm.weight", "offset": 1216911360, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 1216925696, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 1216940032, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1216954368, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 1224753152, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.5.attn_q.weight", "offset": 1240350720, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.5.attn_v.weight", "offset": 1255948288, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.6.attn_norm.weight", "offset": 1263747072, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1263761408, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.6.ffn_gate.weight", "offset": 1318352896, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.6.ffn_up.weight", "offset": 1372944384, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.6.post_attention_norm.weight", "offset": 1427535872, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 1427550208, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 1427564544, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1427578880, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 1435377664, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.6.attn_q.weight", "offset": 1450975232, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.6.attn_v.weight", "offset": 1466572800, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.7.attn_norm.weight", "offset": 1474371584, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 1474385920, "shape": [ 3584, 14336 ], "size": 14336, "type": 8 }, { "name": "blk.7.attn_k.weight", "offset": 1528977408, "shape": [ 3584, 2048 ], "size": 54591488, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 1536776192, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.7.attn_q.weight", "offset": 1552373760, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.7.attn_v.weight", "offset": 1567971328, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.7.ffn_gate.weight", "offset": 1575770112, "shape": [ 3584, 14336 ], "size": 7798784, "type": 8 }, { "name": "blk.7.ffn_down.weight", "offset": 1630361600, "shape": [ 14336, 3584 ], "size": 54591488, "type": 8 }, { "name": "blk.7.post_attention_norm.weight", "offset": 1684953088, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 1684967424, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 1684981760, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 1684996096, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1685010432, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.8.ffn_gate.weight", "offset": 1739601920, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.8.ffn_up.weight", "offset": 1794193408, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 1848784896, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 1848799232, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1848813568, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 1856612352, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.8.attn_q.weight", "offset": 1872209920, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.8.attn_v.weight", "offset": 1887807488, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.8.post_attention_norm.weight", "offset": 1895606272, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1895620608, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.9.ffn_down.weight", "offset": 1903419392, "shape": [ 14336, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 1958010880, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.9.ffn_gate.weight", "offset": 1958025216, "shape": [ 3584, 14336 ], "size": 14336, "type": 8 }, { "name": "blk.9.ffn_up.weight", "offset": 2012616704, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.9.post_attention_norm.weight", "offset": 2067208192, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 2067222528, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 2067236864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2067251200, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 2075049984, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.9.attn_q.weight", "offset": 2090647552, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.10.attn_q.weight", "offset": 2106245120, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.10.attn_v.weight", "offset": 2121842688, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.10.ffn_down.weight", "offset": 2129641472, "shape": [ 14336, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.10.ffn_gate.weight", "offset": 2184232960, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.10.attn_k.weight", "offset": 2238824448, "shape": [ 3584, 2048 ], "size": 54591488, "type": 8 }, { "name": "blk.10.ffn_norm.weight", "offset": 2246623232, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 2246637568, "shape": [ 3584, 14336 ], "size": 14336, "type": 8 }, { "name": "blk.10.post_attention_norm.weight", "offset": 2301229056, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 2301243392, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 2301257728, "shape": [ 4096, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 2316855296, "shape": [ 3584 ], "size": 15597568, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 2316869632, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.post_attention_norm.weight", "offset": 2316883968, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 2316898304, "shape": [ 3584, 4096 ], "size": 14336, "type": 8 }, { "name": "blk.11.ffn_up.weight", "offset": 2332495872, "shape": [ 3584, 14336 ], "size": 15597568, "type": 8 }, { "name": "blk.11.attn_k.weight", "offset": 2387087360, "shape": [ 3584, 2048 ], "size": 54591488, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 2394886144, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 2394900480, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 2394914816, "shape": [ 4096, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.11.attn_v.weight", "offset": 2410512384, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.11.ffn_down.weight", "offset": 2418311168, "shape": [ 14336, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.11.ffn_gate.weight", "offset": 2472902656, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.12.attn_norm.weight", "offset": 2527494144, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2527508480, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.12.attn_v.weight", "offset": 2582099968, "shape": [ 3584, 2048 ], "size": 54591488, "type": 8 }, { "name": "blk.12.attn_q.weight", "offset": 2589898752, "shape": [ 3584, 4096 ], "size": 7798784, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 2605496320, "shape": [ 4096, 3584 ], "size": 15597568, "type": 8 }, { "name": "blk.12.attn_k.weight", "offset": 2621093888, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.12.ffn_norm.weight", "offset": 2628892672, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 2628907008, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.post_attention_norm.weight", "offset": 2628921344, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 2628935680, "shape": [ 3584, 14336 ], "size": 14336, "type": 8 }, { "name": "blk.12.ffn_gate.weight", "offset": 2683527168, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 2738118656, "shape": [ 4096, 3584 ], "size": 54591488, "type": 8 }, { "name": "blk.13.attn_v.weight", "offset": 2753716224, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.13.ffn_norm.weight", "offset": 2761515008, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 2761529344, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.post_attention_norm.weight", "offset": 2761543680, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 2761558016, "shape": [ 3584, 14336 ], "size": 14336, "type": 8 }, { "name": "blk.13.ffn_gate.weight", "offset": 2816149504, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.13.ffn_down.weight", "offset": 2870740992, "shape": [ 14336, 3584 ], "size": 54591488, "type": 8 }, { "name": "blk.13.attn_norm.weight", "offset": 2925332480, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2925346816, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.13.attn_q.weight", "offset": 2933145600, "shape": [ 3584, 4096 ], "size": 7798784, "type": 8 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 2948743168, "shape": [ 3584 ], "size": 15597568, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 2948757504, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2948771840, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.14.ffn_gate.weight", "offset": 3003363328, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.14.ffn_up.weight", "offset": 3057954816, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.14.post_attention_norm.weight", "offset": 3112546304, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 3112560640, "shape": [ 4096, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.14.ffn_norm.weight", "offset": 3128158208, "shape": [ 3584 ], "size": 15597568, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 3128172544, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.14.attn_v.weight", "offset": 3135971328, "shape": [ 3584, 2048 ], "size": 7798784, "type": 8 }, { "name": "blk.14.attn_q.weight", "offset": 3143770112, "shape": [ 3584, 4096 ], "size": 7798784, "type": 8 }, { "name": "blk.15.attn_v.weight", "offset": 3159367680, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.15.attn_q.weight", "offset": 3167166464, "shape": [ 3584, 4096 ], "size": 7798784, "type": 8 }, { "name": "blk.15.attn_norm.weight", "offset": 3182764032, "shape": [ 3584 ], "size": 15597568, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3182778368, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.15.ffn_gate.weight", "offset": 3237369856, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.15.ffn_up.weight", "offset": 3291961344, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.15.post_attention_norm.weight", "offset": 3346552832, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 3346567168, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 3346581504, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 3346595840, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 3354394624, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.16.ffn_norm.weight", "offset": 3369992192, "shape": [ 3584 ], "size": 15597568, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 3370006528, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 3370020864, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 3377819648, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 3377833984, "shape": [ 3584, 4096 ], "size": 14336, "type": 8 }, { "name": "blk.16.attn_v.weight", "offset": 3393431552, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 3401230336, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.16.post_attention_norm.weight", "offset": 3416827904, "shape": [ 3584 ], "size": 15597568, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 3416842240, "shape": [ 3584, 14336 ], "size": 14336, "type": 8 }, { "name": "blk.16.ffn_gate.weight", "offset": 3471433728, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.16.ffn_down.weight", "offset": 3526025216, "shape": [ 14336, 3584 ], "size": 54591488, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 3580616704, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 3580631040, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 3580645376, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 3588444160, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 3604041728, "shape": [ 3584 ], "size": 15597568, "type": 0 }, { "name": "blk.17.post_attention_norm.weight", "offset": 3604056064, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 3604070400, "shape": [ 3584, 14336 ], "size": 14336, "type": 8 }, { "name": "blk.17.ffn_gate.weight", "offset": 3658661888, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.17.ffn_down.weight", "offset": 3713253376, "shape": [ 14336, 3584 ], "size": 54591488, "type": 8 }, { "name": "blk.17.attn_v.weight", "offset": 3767844864, "shape": [ 3584, 2048 ], "size": 54591488, "type": 8 }, { "name": "blk.17.attn_q.weight", "offset": 3775643648, "shape": [ 3584, 4096 ], "size": 7798784, "type": 8 }, { "name": "blk.18.attn_v.weight", "offset": 3791241216, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 3799040000, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 3799054336, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3799068672, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.18.ffn_gate.weight", "offset": 3853660160, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.18.ffn_up.weight", "offset": 3908251648, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.18.post_attention_norm.weight", "offset": 3962843136, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 3962857472, "shape": [ 3584, 4096 ], "size": 14336, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 3978455040, "shape": [ 4096, 3584 ], "size": 15597568, "type": 8 }, { "name": "blk.18.attn_k.weight", "offset": 3994052608, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 4001851392, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 4001865728, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 4001880064, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4001894400, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.19.ffn_gate.weight", "offset": 4056485888, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.19.ffn_up.weight", "offset": 4111077376, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.19.post_attention_norm.weight", "offset": 4165668864, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 4165683200, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 4165697536, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 4173496320, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.19.attn_q.weight", "offset": 4189093888, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.19.attn_v.weight", "offset": 4204691456, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.20.attn_k.weight", "offset": 4212490240, "shape": [ 3584, 2048 ], "size": 7798784, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 4220289024, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.20.attn_q.weight", "offset": 4235886592, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.20.attn_v.weight", "offset": 4251484160, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.20.attn_norm.weight", "offset": 4259282944, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4259297280, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.20.ffn_gate.weight", "offset": 4313888768, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.20.ffn_up.weight", "offset": 4368480256, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.20.post_attention_norm.weight", "offset": 4423071744, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 4423086080, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 4423100416, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4423114752, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.21.attn_norm.weight", "offset": 4477706240, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.21.ffn_gate.weight", "offset": 4477720576, "shape": [ 3584, 14336 ], "size": 14336, "type": 8 }, { "name": "blk.21.ffn_up.weight", "offset": 4532312064, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.21.post_attention_norm.weight", "offset": 4586903552, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 4586917888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 4586932224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 4586946560, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 4594745344, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.21.attn_q.weight", "offset": 4610342912, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.21.attn_v.weight", "offset": 4625940480, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 4633739264, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4633753600, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.22.ffn_gate.weight", "offset": 4688345088, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.22.ffn_up.weight", "offset": 4742936576, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.22.post_attention_norm.weight", "offset": 4797528064, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 4797542400, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 4797556736, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 4797571072, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 4805369856, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.22.attn_q.weight", "offset": 4820967424, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.22.attn_v.weight", "offset": 4836564992, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.23.attn_norm.weight", "offset": 4844363776, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4844378112, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.23.ffn_gate.weight", "offset": 4898969600, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.23.ffn_up.weight", "offset": 4953561088, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.23.post_attention_norm.weight", "offset": 5008152576, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 5008166912, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 5008181248, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 5008195584, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 5015994368, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.23.attn_q.weight", "offset": 5031591936, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.23.attn_v.weight", "offset": 5047189504, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.24.attn_norm.weight", "offset": 5054988288, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5055002624, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.24.ffn_gate.weight", "offset": 5109594112, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.24.ffn_up.weight", "offset": 5164185600, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.24.post_attention_norm.weight", "offset": 5218777088, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 5218791424, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 5218805760, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 5218820096, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 5226618880, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.24.attn_q.weight", "offset": 5242216448, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.24.attn_v.weight", "offset": 5257814016, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.25.attn_norm.weight", "offset": 5265612800, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5265627136, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.25.ffn_gate.weight", "offset": 5320218624, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.25.ffn_up.weight", "offset": 5374810112, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.25.post_attention_norm.weight", "offset": 5429401600, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 5429415936, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 5429430272, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 5429444608, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 5437243392, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.25.attn_q.weight", "offset": 5452840960, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.25.attn_v.weight", "offset": 5468438528, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.26.attn_norm.weight", "offset": 5476237312, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5476251648, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.26.ffn_gate.weight", "offset": 5530843136, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.26.ffn_up.weight", "offset": 5585434624, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.26.post_attention_norm.weight", "offset": 5640026112, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 5640040448, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 5640054784, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 5640069120, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 5647867904, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.26.attn_q.weight", "offset": 5663465472, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.26.attn_v.weight", "offset": 5679063040, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.27.attn_norm.weight", "offset": 5686861824, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5686876160, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.27.ffn_gate.weight", "offset": 5741467648, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.27.ffn_up.weight", "offset": 5796059136, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.27.post_attention_norm.weight", "offset": 5850650624, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 5850664960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 5850679296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 5850693632, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 5858492416, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.27.attn_q.weight", "offset": 5874089984, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.27.attn_v.weight", "offset": 5889687552, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.28.attn_norm.weight", "offset": 5897486336, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5897500672, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.28.ffn_gate.weight", "offset": 5952092160, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.28.ffn_up.weight", "offset": 6006683648, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.28.post_attention_norm.weight", "offset": 6061275136, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 6061289472, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 6061303808, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 6061318144, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 6069116928, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.28.attn_q.weight", "offset": 6084714496, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.28.attn_v.weight", "offset": 6100312064, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.29.attn_norm.weight", "offset": 6108110848, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 6108125184, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.29.ffn_gate.weight", "offset": 6162716672, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.29.ffn_up.weight", "offset": 6217308160, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.29.post_attention_norm.weight", "offset": 6271899648, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 6271913984, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 6271928320, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 6271942656, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 6279741440, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.29.attn_q.weight", "offset": 6295339008, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.29.attn_v.weight", "offset": 6310936576, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.30.attn_norm.weight", "offset": 6318735360, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 6318749696, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.30.ffn_gate.weight", "offset": 6373341184, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.30.ffn_up.weight", "offset": 6427932672, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.30.post_attention_norm.weight", "offset": 6482524160, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 6482538496, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 6482552832, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 6482567168, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 6490365952, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.30.attn_q.weight", "offset": 6505963520, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.30.attn_v.weight", "offset": 6521561088, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.31.attn_k.weight", "offset": 6529359872, "shape": [ 3584, 2048 ], "size": 7798784, "type": 8 }, { "name": "blk.31.ffn_down.weight", "offset": 6537158656, "shape": [ 14336, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.31.ffn_gate.weight", "offset": 6591750144, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.31.ffn_up.weight", "offset": 6646341632, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.31.post_attention_norm.weight", "offset": 6700933120, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 6700947456, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 6700961792, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 6700976128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 6700990464, "shape": [ 4096, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.31.attn_q.weight", "offset": 6716588032, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.31.attn_v.weight", "offset": 6732185600, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.32.ffn_gate.weight", "offset": 6739984384, "shape": [ 3584, 14336 ], "size": 7798784, "type": 8 }, { "name": "blk.32.ffn_up.weight", "offset": 6794575872, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.32.attn_k.weight", "offset": 6849167360, "shape": [ 3584, 2048 ], "size": 54591488, "type": 8 }, { "name": "blk.32.attn_output.weight", "offset": 6856966144, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.32.attn_q.weight", "offset": 6872563712, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.32.attn_v.weight", "offset": 6888161280, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.32.attn_norm.weight", "offset": 6895960064, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 6895974400, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.32.post_attention_norm.weight", "offset": 6950565888, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 6950580224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 6950594560, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 6950608896, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6950623232, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.33.ffn_gate.weight", "offset": 7005214720, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.33.ffn_up.weight", "offset": 7059806208, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.33.post_attention_norm.weight", "offset": 7114397696, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 7114412032, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 7114426368, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 7114440704, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.33.attn_output.weight", "offset": 7122239488, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.33.attn_q.weight", "offset": 7137837056, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.33.attn_v.weight", "offset": 7153434624, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.34.attn_norm.weight", "offset": 7161233408, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 7161247744, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.34.ffn_gate.weight", "offset": 7215839232, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.34.ffn_up.weight", "offset": 7270430720, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.34.post_attention_norm.weight", "offset": 7325022208, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 7325036544, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 7325050880, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 7325065216, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.34.attn_output.weight", "offset": 7332864000, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.34.attn_q.weight", "offset": 7348461568, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.34.attn_v.weight", "offset": 7364059136, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.35.attn_norm.weight", "offset": 7371857920, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 7371872256, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.35.ffn_gate.weight", "offset": 7426463744, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.35.ffn_up.weight", "offset": 7481055232, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.35.post_attention_norm.weight", "offset": 7535646720, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 7535661056, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 7535675392, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 7535689728, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.35.attn_output.weight", "offset": 7543488512, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.35.attn_q.weight", "offset": 7559086080, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.35.attn_v.weight", "offset": 7574683648, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.36.ffn_norm.weight", "offset": 7582482432, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 7582496768, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.36.ffn_gate.weight", "offset": 7637088256, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.36.ffn_up.weight", "offset": 7691679744, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.36.post_attention_norm.weight", "offset": 7746271232, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 7746285568, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 7746299904, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 7746314240, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.36.attn_output.weight", "offset": 7754113024, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.36.attn_q.weight", "offset": 7769710592, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.36.attn_v.weight", "offset": 7785308160, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.37.attn_norm.weight", "offset": 7793106944, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 7793121280, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.37.ffn_gate.weight", "offset": 7847712768, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.37.ffn_up.weight", "offset": 7902304256, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.37.post_attention_norm.weight", "offset": 7956895744, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 7956910080, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 7956924416, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 7956938752, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.37.attn_output.weight", "offset": 7964737536, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.37.attn_q.weight", "offset": 7980335104, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.37.attn_v.weight", "offset": 7995932672, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.38.attn_norm.weight", "offset": 8003731456, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 8003745792, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.38.ffn_gate.weight", "offset": 8058337280, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.38.ffn_up.weight", "offset": 8112928768, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.38.post_attention_norm.weight", "offset": 8167520256, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 8167534592, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 8167548928, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 8167563264, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.38.attn_output.weight", "offset": 8175362048, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.38.attn_q.weight", "offset": 8190959616, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.38.attn_v.weight", "offset": 8206557184, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.39.attn_norm.weight", "offset": 8214355968, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 8214370304, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.39.ffn_gate.weight", "offset": 8268961792, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.39.ffn_up.weight", "offset": 8323553280, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.39.post_attention_norm.weight", "offset": 8378144768, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 8378159104, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 8378173440, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 8378187776, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.39.attn_output.weight", "offset": 8385986560, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.39.attn_q.weight", "offset": 8401584128, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.39.attn_v.weight", "offset": 8417181696, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.40.attn_norm.weight", "offset": 8424980480, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 8424994816, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.40.ffn_gate.weight", "offset": 8479586304, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.40.ffn_up.weight", "offset": 8534177792, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.40.post_attention_norm.weight", "offset": 8588769280, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 8588783616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 8588797952, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 8588812288, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.40.attn_output.weight", "offset": 8596611072, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.40.attn_q.weight", "offset": 8612208640, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.40.attn_v.weight", "offset": 8627806208, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "blk.41.attn_norm.weight", "offset": 8635604992, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 8635619328, "shape": [ 14336, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.41.ffn_gate.weight", "offset": 8690210816, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.41.ffn_up.weight", "offset": 8744802304, "shape": [ 3584, 14336 ], "size": 54591488, "type": 8 }, { "name": "blk.41.post_attention_norm.weight", "offset": 8799393792, "shape": [ 3584 ], "size": 54591488, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 8799408128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 8799422464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 8799436800, "shape": [ 3584, 2048 ], "size": 14336, "type": 8 }, { "name": "blk.41.attn_output.weight", "offset": 8807235584, "shape": [ 4096, 3584 ], "size": 7798784, "type": 8 }, { "name": "blk.41.attn_q.weight", "offset": 8822833152, "shape": [ 3584, 4096 ], "size": 15597568, "type": 8 }, { "name": "blk.41.attn_v.weight", "offset": 8838430720, "shape": [ 3584, 2048 ], "size": 15597568, "type": 8 }, { "name": "output_norm.weight", "offset": 8846229504, "shape": [ 3584 ], "size": 7798784, "type": 0 }, { "name": "token_embd.weight", "offset": 8846243840, "shape": [ 3584, 256000 ], "size": 14336, "type": 8 } ], "version": 3 }