gemma2

{ "metadata": { "gemma2.attention.head_count": 32, "gemma2.attention.head_count_kv": 16, "gemma2.attention.key_length": 128, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 128, "gemma2.block_count": 46, "gemma2.context_length": 8192, "gemma2.embedding_length": 4608, "gemma2.feed_forward_length": 36864, "general.architecture": "gemma2", "general.file_type": 8, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 27227128320, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 18432, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.0.ffn_gate.weight", "offset": 116803584, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.0.ffn_up.weight", "offset": 233588736, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.0.post_attention_norm.weight", "offset": 350373888, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 350392320, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 350410752, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 350429184, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.0.attn_output.weight", "offset": 356917248, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.0.attn_q.weight", "offset": 369893376, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.0.attn_v.weight", "offset": 382869504, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.1.attn_norm.weight", "offset": 389357568, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 389376000, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.1.ffn_gate.weight", "offset": 506161152, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.1.ffn_up.weight", "offset": 622946304, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.1.post_attention_norm.weight", "offset": 739731456, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 739749888, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 739768320, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 739786752, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.1.attn_output.weight", "offset": 746274816, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.1.attn_q.weight", "offset": 759250944, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.1.attn_v.weight", "offset": 772227072, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.2.attn_k.weight", "offset": 778715136, "shape": [ 4608, 2048 ], "size": 6488064, "type": 6 }, { "name": "blk.2.attn_output.weight", "offset": 785203200, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.2.attn_q.weight", "offset": 798179328, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.2.attn_v.weight", "offset": 811155456, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.2.attn_norm.weight", "offset": 817643520, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 817661952, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.2.ffn_gate.weight", "offset": 934447104, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.2.ffn_up.weight", "offset": 1051232256, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.2.post_attention_norm.weight", "offset": 1168017408, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 1168035840, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 1168054272, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_norm.weight", "offset": 1168072704, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1168091136, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.3.ffn_gate.weight", "offset": 1284876288, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.3.ffn_up.weight", "offset": 1401661440, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.3.post_attention_norm.weight", "offset": 1518446592, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 1518465024, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 1518483456, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1518501888, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.3.attn_output.weight", "offset": 1524989952, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.3.attn_q.weight", "offset": 1537966080, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.3.attn_v.weight", "offset": 1550942208, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.4.attn_norm.weight", "offset": 1557430272, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1557448704, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.4.ffn_gate.weight", "offset": 1674233856, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.4.ffn_up.weight", "offset": 1791019008, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.4.post_attention_norm.weight", "offset": 1907804160, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 1907822592, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 1907841024, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1907859456, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.4.attn_output.weight", "offset": 1914347520, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.4.attn_q.weight", "offset": 1927323648, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.4.attn_v.weight", "offset": 1940299776, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.5.attn_norm.weight", "offset": 1946787840, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1946806272, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.5.ffn_gate.weight", "offset": 2063591424, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.5.ffn_up.weight", "offset": 2180376576, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.5.post_attention_norm.weight", "offset": 2297161728, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 2297180160, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 2297198592, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 2297217024, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.5.attn_output.weight", "offset": 2303705088, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.5.attn_q.weight", "offset": 2316681216, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.5.attn_v.weight", "offset": 2329657344, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.6.ffn_gate.weight", "offset": 2336145408, "shape": [ 4608, 36864 ], "size": 6488064, "type": 6 }, { "name": "blk.6.attn_k.weight", "offset": 2452930560, "shape": [ 4608, 2048 ], "size": 116785152, "type": 6 }, { "name": "blk.6.attn_output.weight", "offset": 2459418624, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.6.attn_q.weight", "offset": 2472394752, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.6.attn_v.weight", "offset": 2485370880, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.6.attn_norm.weight", "offset": 2491858944, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2491877376, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.6.ffn_up.weight", "offset": 2608662528, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.6.post_attention_norm.weight", "offset": 2725447680, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 2725466112, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 2725484544, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2725502976, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2725521408, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.7.ffn_gate.weight", "offset": 2842306560, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.7.ffn_up.weight", "offset": 2959091712, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.7.post_attention_norm.weight", "offset": 3075876864, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 3075895296, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 3075913728, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 3075932160, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.7.attn_output.weight", "offset": 3082420224, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.7.attn_q.weight", "offset": 3095396352, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.7.attn_v.weight", "offset": 3108372480, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.8.attn_norm.weight", "offset": 3114860544, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3114878976, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.8.ffn_gate.weight", "offset": 3231664128, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.8.ffn_up.weight", "offset": 3348449280, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.8.post_attention_norm.weight", "offset": 3465234432, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 3465252864, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 3465271296, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 3465289728, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.8.attn_output.weight", "offset": 3471777792, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.8.attn_q.weight", "offset": 3484753920, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.8.attn_v.weight", "offset": 3497730048, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.9.attn_norm.weight", "offset": 3504218112, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3504236544, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.9.ffn_gate.weight", "offset": 3621021696, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.9.ffn_up.weight", "offset": 3737806848, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.9.post_attention_norm.weight", "offset": 3854592000, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 3854610432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 3854628864, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 3854647296, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.9.attn_output.weight", "offset": 3861135360, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.9.attn_q.weight", "offset": 3874111488, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.9.attn_v.weight", "offset": 3887087616, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.10.ffn_gate.weight", "offset": 3893575680, "shape": [ 4608, 36864 ], "size": 6488064, "type": 6 }, { "name": "blk.10.ffn_up.weight", "offset": 4010360832, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.10.attn_k.weight", "offset": 4127145984, "shape": [ 4608, 2048 ], "size": 116785152, "type": 6 }, { "name": "blk.10.attn_output.weight", "offset": 4133634048, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.10.attn_q.weight", "offset": 4146610176, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.10.attn_v.weight", "offset": 4159586304, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.10.attn_norm.weight", "offset": 4166074368, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 4166092800, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.10.post_attention_norm.weight", "offset": 4282877952, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 4282896384, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 4282914816, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_norm.weight", "offset": 4282933248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 4282951680, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.11.ffn_gate.weight", "offset": 4399736832, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.11.ffn_up.weight", "offset": 4516521984, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.11.post_attention_norm.weight", "offset": 4633307136, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 4633325568, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 4633344000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 4633362432, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.11.attn_output.weight", "offset": 4639850496, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.11.attn_q.weight", "offset": 4652826624, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.11.attn_v.weight", "offset": 4665802752, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.12.attn_norm.weight", "offset": 4672290816, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 4672309248, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.12.ffn_gate.weight", "offset": 4789094400, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.12.ffn_up.weight", "offset": 4905879552, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.12.post_attention_norm.weight", "offset": 5022664704, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 5022683136, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 5022701568, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 5022720000, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.12.attn_output.weight", "offset": 5029208064, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.12.attn_q.weight", "offset": 5042184192, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.12.attn_v.weight", "offset": 5055160320, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.13.attn_norm.weight", "offset": 5061648384, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 5061666816, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.13.ffn_gate.weight", "offset": 5178451968, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.13.ffn_up.weight", "offset": 5295237120, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.13.post_attention_norm.weight", "offset": 5412022272, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 5412040704, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 5412059136, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 5412077568, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.13.attn_output.weight", "offset": 5418565632, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.13.attn_q.weight", "offset": 5431541760, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.13.attn_v.weight", "offset": 5444517888, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.14.attn_norm.weight", "offset": 5451005952, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 5451024384, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.14.ffn_gate.weight", "offset": 5567809536, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.14.ffn_up.weight", "offset": 5684594688, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.14.post_attention_norm.weight", "offset": 5801379840, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 5801398272, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 5801416704, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 5801435136, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.14.attn_output.weight", "offset": 5807923200, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.14.attn_q.weight", "offset": 5820899328, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.14.attn_v.weight", "offset": 5833875456, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.15.attn_k.weight", "offset": 5840363520, "shape": [ 4608, 2048 ], "size": 6488064, "type": 6 }, { "name": "blk.15.attn_output.weight", "offset": 5846851584, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.15.attn_q.weight", "offset": 5859827712, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.15.attn_v.weight", "offset": 5872803840, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.15.attn_norm.weight", "offset": 5879291904, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 5879310336, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.15.ffn_gate.weight", "offset": 5996095488, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.15.ffn_up.weight", "offset": 6112880640, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.15.post_attention_norm.weight", "offset": 6229665792, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 6229684224, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 6229702656, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 6229721088, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 6229739520, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.16.ffn_gate.weight", "offset": 6346524672, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.16.ffn_up.weight", "offset": 6463309824, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.16.post_attention_norm.weight", "offset": 6580094976, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 6580113408, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 6580131840, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 6580150272, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.16.attn_output.weight", "offset": 6586638336, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.16.attn_q.weight", "offset": 6599614464, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.16.attn_v.weight", "offset": 6612590592, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.17.attn_norm.weight", "offset": 6619078656, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 6619097088, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.17.ffn_gate.weight", "offset": 6735882240, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.17.ffn_up.weight", "offset": 6852667392, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.17.post_attention_norm.weight", "offset": 6969452544, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 6969470976, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 6969489408, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 6969507840, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.17.attn_output.weight", "offset": 6975995904, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.17.attn_q.weight", "offset": 6988972032, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.17.attn_v.weight", "offset": 7001948160, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.18.attn_norm.weight", "offset": 7008436224, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 7008454656, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.18.ffn_gate.weight", "offset": 7125239808, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.18.ffn_up.weight", "offset": 7242024960, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.18.post_attention_norm.weight", "offset": 7358810112, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 7358828544, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 7358846976, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 7358865408, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.18.attn_output.weight", "offset": 7365353472, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.18.attn_q.weight", "offset": 7378329600, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.18.attn_v.weight", "offset": 7391305728, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.19.ffn_gate.weight", "offset": 7397793792, "shape": [ 4608, 36864 ], "size": 6488064, "type": 6 }, { "name": "blk.19.attn_k.weight", "offset": 7514578944, "shape": [ 4608, 2048 ], "size": 116785152, "type": 6 }, { "name": "blk.19.attn_output.weight", "offset": 7521067008, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.19.attn_q.weight", "offset": 7534043136, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.19.attn_v.weight", "offset": 7547019264, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.19.attn_norm.weight", "offset": 7553507328, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 7553525760, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.19.ffn_up.weight", "offset": 7670310912, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.19.post_attention_norm.weight", "offset": 7787096064, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 7787114496, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 7787132928, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_norm.weight", "offset": 7787151360, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 7787169792, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.20.ffn_gate.weight", "offset": 7903954944, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.20.ffn_up.weight", "offset": 8020740096, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.20.post_attention_norm.weight", "offset": 8137525248, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 8137543680, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 8137562112, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 8137580544, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.20.attn_output.weight", "offset": 8144068608, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.20.attn_q.weight", "offset": 8157044736, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.20.attn_v.weight", "offset": 8170020864, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.21.attn_norm.weight", "offset": 8176508928, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 8176527360, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.21.ffn_gate.weight", "offset": 8293312512, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.21.ffn_up.weight", "offset": 8410097664, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.21.post_attention_norm.weight", "offset": 8526882816, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 8526901248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 8526919680, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 8526938112, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.21.attn_output.weight", "offset": 8533426176, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.21.attn_q.weight", "offset": 8546402304, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.21.attn_v.weight", "offset": 8559378432, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.22.attn_norm.weight", "offset": 8565866496, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 8565884928, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.22.ffn_gate.weight", "offset": 8682670080, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.22.ffn_up.weight", "offset": 8799455232, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.22.post_attention_norm.weight", "offset": 8916240384, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 8916258816, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 8916277248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 8916295680, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.22.attn_output.weight", "offset": 8922783744, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.22.attn_q.weight", "offset": 8935759872, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.22.attn_v.weight", "offset": 8948736000, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.23.ffn_up.weight", "offset": 8955224064, "shape": [ 4608, 36864 ], "size": 6488064, "type": 6 }, { "name": "blk.23.ffn_gate.weight", "offset": 9072009216, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.23.attn_k.weight", "offset": 9188794368, "shape": [ 4608, 2048 ], "size": 116785152, "type": 6 }, { "name": "blk.23.attn_output.weight", "offset": 9195282432, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.23.attn_q.weight", "offset": 9208258560, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.23.attn_v.weight", "offset": 9221234688, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.23.attn_norm.weight", "offset": 9227722752, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 9227741184, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.23.post_attention_norm.weight", "offset": 9344526336, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 9344544768, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 9344563200, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 9344581632, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 9344600064, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.24.ffn_gate.weight", "offset": 9461385216, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.24.ffn_up.weight", "offset": 9578170368, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.24.post_attention_norm.weight", "offset": 9694955520, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 9694973952, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 9694992384, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 9695010816, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.24.attn_output.weight", "offset": 9701498880, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.24.attn_q.weight", "offset": 9714475008, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.24.attn_v.weight", "offset": 9727451136, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.25.attn_norm.weight", "offset": 9733939200, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 9733957632, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.25.ffn_gate.weight", "offset": 9850742784, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.25.ffn_up.weight", "offset": 9967527936, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.25.post_attention_norm.weight", "offset": 10084313088, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 10084331520, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 10084349952, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 10084368384, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.25.attn_output.weight", "offset": 10090856448, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.25.attn_q.weight", "offset": 10103832576, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.25.attn_v.weight", "offset": 10116808704, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.26.attn_norm.weight", "offset": 10123296768, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 10123315200, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.26.ffn_gate.weight", "offset": 10240100352, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.26.ffn_up.weight", "offset": 10356885504, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.26.post_attention_norm.weight", "offset": 10473670656, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 10473689088, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 10473707520, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 10473725952, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.26.attn_output.weight", "offset": 10480214016, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.26.attn_q.weight", "offset": 10493190144, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.26.attn_v.weight", "offset": 10506166272, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.27.attn_norm.weight", "offset": 10512654336, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 10512672768, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.27.ffn_gate.weight", "offset": 10629457920, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.27.ffn_up.weight", "offset": 10746243072, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.27.post_attention_norm.weight", "offset": 10863028224, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 10863046656, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 10863065088, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 10863083520, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.27.attn_output.weight", "offset": 10869571584, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.27.attn_q.weight", "offset": 10882547712, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.27.attn_v.weight", "offset": 10895523840, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.28.attn_k.weight", "offset": 10902011904, "shape": [ 4608, 2048 ], "size": 6488064, "type": 6 }, { "name": "blk.28.attn_output.weight", "offset": 10908499968, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.28.attn_q.weight", "offset": 10921476096, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.28.attn_v.weight", "offset": 10934452224, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.28.attn_norm.weight", "offset": 10940940288, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 10940958720, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.28.ffn_gate.weight", "offset": 11057743872, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.28.ffn_up.weight", "offset": 11174529024, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.28.post_attention_norm.weight", "offset": 11291314176, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 11291332608, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 11291351040, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 11291369472, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 11291387904, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.29.ffn_gate.weight", "offset": 11408173056, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.29.ffn_up.weight", "offset": 11524958208, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.29.post_attention_norm.weight", "offset": 11641743360, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 11641761792, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 11641780224, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 11641798656, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.29.attn_output.weight", "offset": 11648286720, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.29.attn_q.weight", "offset": 11661262848, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.29.attn_v.weight", "offset": 11674238976, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.30.attn_norm.weight", "offset": 11680727040, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 11680745472, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.30.ffn_gate.weight", "offset": 11797530624, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.30.ffn_up.weight", "offset": 11914315776, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.30.post_attention_norm.weight", "offset": 12031100928, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 12031119360, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 12031137792, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 12031156224, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.30.attn_output.weight", "offset": 12037644288, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.30.attn_q.weight", "offset": 12050620416, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.30.attn_v.weight", "offset": 12063596544, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.31.attn_norm.weight", "offset": 12070084608, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 12070103040, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.31.ffn_gate.weight", "offset": 12186888192, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.31.ffn_up.weight", "offset": 12303673344, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.31.post_attention_norm.weight", "offset": 12420458496, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 12420476928, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 12420495360, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 12420513792, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.31.attn_output.weight", "offset": 12427001856, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.31.attn_q.weight", "offset": 12439977984, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.31.attn_v.weight", "offset": 12452954112, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.32.ffn_gate.weight", "offset": 12459442176, "shape": [ 4608, 36864 ], "size": 6488064, "type": 6 }, { "name": "blk.32.attn_k.weight", "offset": 12576227328, "shape": [ 4608, 2048 ], "size": 116785152, "type": 6 }, { "name": "blk.32.attn_output.weight", "offset": 12582715392, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.32.attn_q.weight", "offset": 12595691520, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.32.attn_v.weight", "offset": 12608667648, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.32.attn_norm.weight", "offset": 12615155712, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 12615174144, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.32.ffn_up.weight", "offset": 12731959296, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.32.post_attention_norm.weight", "offset": 12848744448, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 12848762880, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 12848781312, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 12848799744, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 12848818176, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.33.ffn_gate.weight", "offset": 12965603328, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.33.ffn_up.weight", "offset": 13082388480, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.33.post_attention_norm.weight", "offset": 13199173632, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 13199192064, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 13199210496, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 13199228928, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.33.attn_output.weight", "offset": 13205716992, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.33.attn_q.weight", "offset": 13218693120, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.33.attn_v.weight", "offset": 13231669248, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.34.ffn_norm.weight", "offset": 13238157312, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 13238175744, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.34.ffn_gate.weight", "offset": 13354960896, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.34.ffn_up.weight", "offset": 13471746048, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.34.post_attention_norm.weight", "offset": 13588531200, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 13588549632, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_norm.weight", "offset": 13588568064, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 13588586496, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.34.attn_output.weight", "offset": 13595074560, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.34.attn_q.weight", "offset": 13608050688, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.34.attn_v.weight", "offset": 13621026816, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.35.attn_norm.weight", "offset": 13627514880, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 13627533312, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.35.ffn_gate.weight", "offset": 13744318464, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.35.ffn_up.weight", "offset": 13861103616, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.35.post_attention_norm.weight", "offset": 13977888768, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 13977907200, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 13977925632, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 13977944064, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.35.attn_output.weight", "offset": 13984432128, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.35.attn_q.weight", "offset": 13997408256, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.35.attn_v.weight", "offset": 14010384384, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.36.ffn_gate.weight", "offset": 14016872448, "shape": [ 4608, 36864 ], "size": 6488064, "type": 6 }, { "name": "blk.36.ffn_up.weight", "offset": 14133657600, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.36.attn_k.weight", "offset": 14250442752, "shape": [ 4608, 2048 ], "size": 116785152, "type": 6 }, { "name": "blk.36.attn_output.weight", "offset": 14256930816, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.36.attn_q.weight", "offset": 14269906944, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.36.attn_v.weight", "offset": 14282883072, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.36.attn_norm.weight", "offset": 14289371136, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 14289389568, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.36.post_attention_norm.weight", "offset": 14406174720, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 14406193152, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 14406211584, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 14406230016, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 14406248448, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.37.ffn_gate.weight", "offset": 14523033600, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.37.ffn_up.weight", "offset": 14639818752, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.37.post_attention_norm.weight", "offset": 14756603904, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 14756622336, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 14756640768, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 14756659200, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.37.attn_output.weight", "offset": 14763147264, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.37.attn_q.weight", "offset": 14776123392, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.37.attn_v.weight", "offset": 14789099520, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.38.attn_norm.weight", "offset": 14795587584, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 14795606016, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.38.ffn_gate.weight", "offset": 14912391168, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.38.ffn_up.weight", "offset": 15029176320, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.38.post_attention_norm.weight", "offset": 15145961472, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 15145979904, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 15145998336, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 15146016768, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.38.attn_output.weight", "offset": 15152504832, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.38.attn_q.weight", "offset": 15165480960, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.38.attn_v.weight", "offset": 15178457088, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.39.attn_norm.weight", "offset": 15184945152, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 15184963584, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.39.ffn_gate.weight", "offset": 15301748736, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.39.ffn_up.weight", "offset": 15418533888, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.39.post_attention_norm.weight", "offset": 15535319040, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 15535337472, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 15535355904, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 15535374336, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.39.attn_output.weight", "offset": 15541862400, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.39.attn_q.weight", "offset": 15554838528, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.39.attn_v.weight", "offset": 15567814656, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.40.ffn_down.weight", "offset": 15574302720, "shape": [ 36864, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.40.attn_norm.weight", "offset": 15691087872, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.40.ffn_gate.weight", "offset": 15691106304, "shape": [ 4608, 36864 ], "size": 18432, "type": 6 }, { "name": "blk.40.ffn_up.weight", "offset": 15807891456, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.40.post_attention_norm.weight", "offset": 15924676608, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 15924695040, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 15924713472, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 15924731904, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.40.attn_output.weight", "offset": 15931219968, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.40.attn_q.weight", "offset": 15944196096, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.40.attn_v.weight", "offset": 15957172224, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.41.attn_k.weight", "offset": 15963660288, "shape": [ 4608, 2048 ], "size": 6488064, "type": 6 }, { "name": "blk.41.attn_output.weight", "offset": 15970148352, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.41.attn_q.weight", "offset": 15983124480, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.41.attn_v.weight", "offset": 15996100608, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.41.attn_norm.weight", "offset": 16002588672, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 16002607104, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.41.ffn_gate.weight", "offset": 16119392256, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.41.ffn_up.weight", "offset": 16236177408, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.41.post_attention_norm.weight", "offset": 16352962560, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 16352980992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 16352999424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 16353017856, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 16353036288, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.42.ffn_gate.weight", "offset": 16469821440, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.42.ffn_up.weight", "offset": 16586606592, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.42.post_attention_norm.weight", "offset": 16703391744, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.42.post_ffw_norm.weight", "offset": 16703410176, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 16703428608, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 16703447040, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.42.attn_output.weight", "offset": 16709935104, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.42.attn_q.weight", "offset": 16722911232, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.42.attn_v.weight", "offset": 16735887360, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.43.attn_norm.weight", "offset": 16742375424, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 16742393856, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.43.ffn_gate.weight", "offset": 16859179008, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.43.ffn_up.weight", "offset": 16975964160, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.43.post_attention_norm.weight", "offset": 17092749312, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.43.post_ffw_norm.weight", "offset": 17092767744, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 17092786176, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 17092804608, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.43.attn_output.weight", "offset": 17099292672, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.43.attn_q.weight", "offset": 17112268800, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.43.attn_v.weight", "offset": 17125244928, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.44.attn_norm.weight", "offset": 17131732992, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 17131751424, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.44.ffn_gate.weight", "offset": 17248536576, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.44.ffn_up.weight", "offset": 17365321728, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.44.post_attention_norm.weight", "offset": 17482106880, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.44.post_ffw_norm.weight", "offset": 17482125312, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 17482143744, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 17482162176, "shape": [ 4608, 2048 ], "size": 18432, "type": 6 }, { "name": "blk.44.attn_output.weight", "offset": 17488650240, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.44.attn_q.weight", "offset": 17501626368, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.44.attn_v.weight", "offset": 17514602496, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.45.ffn_gate.weight", "offset": 17521090560, "shape": [ 4608, 36864 ], "size": 6488064, "type": 6 }, { "name": "blk.45.attn_k.weight", "offset": 17637875712, "shape": [ 4608, 2048 ], "size": 116785152, "type": 6 }, { "name": "blk.45.attn_output.weight", "offset": 17644363776, "shape": [ 4096, 4608 ], "size": 6488064, "type": 6 }, { "name": "blk.45.attn_q.weight", "offset": 17657339904, "shape": [ 4608, 4096 ], "size": 12976128, "type": 6 }, { "name": "blk.45.attn_v.weight", "offset": 17670316032, "shape": [ 4608, 2048 ], "size": 12976128, "type": 6 }, { "name": "blk.45.attn_norm.weight", "offset": 17676804096, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 17676822528, "shape": [ 36864, 4608 ], "size": 18432, "type": 6 }, { "name": "blk.45.ffn_up.weight", "offset": 17793607680, "shape": [ 4608, 36864 ], "size": 116785152, "type": 6 }, { "name": "blk.45.post_attention_norm.weight", "offset": 17910392832, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.45.post_ffw_norm.weight", "offset": 17910411264, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 17910429696, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "output_norm.weight", "offset": 17910448128, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "token_embd.weight", "offset": 17910466560, "shape": [ 4608, 256000 ], "size": 18432, "type": 14 } ], "version": 3 }