gemma2

{ "metadata": { "gemma2.attention.head_count": 16, "gemma2.attention.head_count_kv": 8, "gemma2.attention.key_length": 256, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 256, "gemma2.block_count": 42, "gemma2.context_length": 8192, "gemma2.embedding_length": 3584, "gemma2.feed_forward_length": 14336, "general.architecture": "gemma2", "general.file_type": 3, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 9241705984, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 14336, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.0.ffn_gate.weight", "offset": 32126976, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.0.ffn_up.weight", "offset": 64239616, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.0.post_attention_norm.weight", "offset": 96352256, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 96366592, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 96380928, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 96395264, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.0.attn_output.weight", "offset": 100982784, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.0.attn_q.weight", "offset": 110157824, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.0.attn_v.weight", "offset": 119332864, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.1.attn_norm.weight", "offset": 123920384, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 123934720, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.1.ffn_gate.weight", "offset": 156047360, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.1.ffn_up.weight", "offset": 188160000, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.1.post_attention_norm.weight", "offset": 220272640, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 220286976, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 220301312, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 220315648, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.1.attn_output.weight", "offset": 224903168, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.1.attn_q.weight", "offset": 234078208, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.1.attn_v.weight", "offset": 243253248, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.2.attn_v.weight", "offset": 247840768, "shape": [ 3584, 2048 ], "size": 4587520, "type": 3 }, { "name": "blk.2.ffn_down.weight", "offset": 252428288, "shape": [ 14336, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.2.ffn_gate.weight", "offset": 284540928, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.2.ffn_up.weight", "offset": 316653568, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.2.post_attention_norm.weight", "offset": 348766208, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 348780544, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 348794880, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 348809216, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.2.attn_output.weight", "offset": 353396736, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.2.attn_q.weight", "offset": 362571776, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.2.attn_norm.weight", "offset": 371746816, "shape": [ 3584 ], "size": 9175040, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 371761152, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.3.ffn_down.weight", "offset": 376348672, "shape": [ 14336, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.3.ffn_gate.weight", "offset": 408461312, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.3.ffn_up.weight", "offset": 440573952, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.3.post_attention_norm.weight", "offset": 472686592, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 472700928, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 472715264, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 472729600, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.3.attn_output.weight", "offset": 477317120, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.3.attn_q.weight", "offset": 486492160, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.3.attn_norm.weight", "offset": 495667200, "shape": [ 3584 ], "size": 9175040, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 495681536, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 495695872, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.4.ffn_gate.weight", "offset": 527808512, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.4.ffn_up.weight", "offset": 559921152, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.4.post_attention_norm.weight", "offset": 592033792, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 592048128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 592062464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 592076800, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.4.attn_output.weight", "offset": 596664320, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.4.attn_q.weight", "offset": 605839360, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.4.attn_v.weight", "offset": 615014400, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.5.attn_norm.weight", "offset": 619601920, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 619616256, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.5.ffn_gate.weight", "offset": 651728896, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.5.ffn_up.weight", "offset": 683841536, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.5.post_attention_norm.weight", "offset": 715954176, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 715968512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 715982848, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 715997184, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.5.attn_output.weight", "offset": 720584704, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.5.attn_q.weight", "offset": 729759744, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.5.attn_v.weight", "offset": 738934784, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.6.attn_norm.weight", "offset": 743522304, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 743536640, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.6.ffn_gate.weight", "offset": 775649280, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.6.ffn_up.weight", "offset": 807761920, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.6.post_attention_norm.weight", "offset": 839874560, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 839888896, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 839903232, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 839917568, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.6.attn_output.weight", "offset": 844505088, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.6.attn_q.weight", "offset": 853680128, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.6.attn_v.weight", "offset": 862855168, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.7.attn_norm.weight", "offset": 867442688, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 867457024, "shape": [ 3584, 14336 ], "size": 14336, "type": 3 }, { "name": "blk.7.attn_k.weight", "offset": 899569664, "shape": [ 3584, 2048 ], "size": 32112640, "type": 3 }, { "name": "blk.7.attn_output.weight", "offset": 904157184, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.7.attn_q.weight", "offset": 913332224, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.7.attn_v.weight", "offset": 922507264, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.7.ffn_gate.weight", "offset": 927094784, "shape": [ 3584, 14336 ], "size": 4587520, "type": 3 }, { "name": "blk.7.ffn_down.weight", "offset": 959207424, "shape": [ 14336, 3584 ], "size": 32112640, "type": 3 }, { "name": "blk.7.post_attention_norm.weight", "offset": 991320064, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 991334400, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 991348736, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 991363072, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 991377408, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.8.ffn_gate.weight", "offset": 1023490048, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.8.ffn_up.weight", "offset": 1055602688, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 1087715328, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 1087729664, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1087744000, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.8.attn_output.weight", "offset": 1092331520, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.8.attn_q.weight", "offset": 1101506560, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.8.attn_v.weight", "offset": 1110681600, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.8.post_attention_norm.weight", "offset": 1115269120, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1115283456, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.9.ffn_down.weight", "offset": 1119870976, "shape": [ 14336, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.9.attn_norm.weight", "offset": 1151983616, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.9.ffn_gate.weight", "offset": 1151997952, "shape": [ 3584, 14336 ], "size": 14336, "type": 3 }, { "name": "blk.9.ffn_up.weight", "offset": 1184110592, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.9.post_attention_norm.weight", "offset": 1216223232, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 1216237568, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 1216251904, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1216266240, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.9.attn_output.weight", "offset": 1220853760, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.9.attn_q.weight", "offset": 1230028800, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.10.attn_q.weight", "offset": 1239203840, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.10.attn_v.weight", "offset": 1248378880, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.10.ffn_down.weight", "offset": 1252966400, "shape": [ 14336, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.10.ffn_gate.weight", "offset": 1285079040, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.10.attn_k.weight", "offset": 1317191680, "shape": [ 3584, 2048 ], "size": 32112640, "type": 3 }, { "name": "blk.10.ffn_norm.weight", "offset": 1321779200, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 1321793536, "shape": [ 3584, 14336 ], "size": 14336, "type": 3 }, { "name": "blk.10.post_attention_norm.weight", "offset": 1353906176, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 1353920512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1353934848, "shape": [ 4096, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.10.attn_norm.weight", "offset": 1363109888, "shape": [ 3584 ], "size": 9175040, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1363124224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.post_attention_norm.weight", "offset": 1363138560, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1363152896, "shape": [ 3584, 4096 ], "size": 14336, "type": 3 }, { "name": "blk.11.ffn_up.weight", "offset": 1372327936, "shape": [ 3584, 14336 ], "size": 9175040, "type": 3 }, { "name": "blk.11.attn_k.weight", "offset": 1404440576, "shape": [ 3584, 2048 ], "size": 32112640, "type": 3 }, { "name": "blk.11.attn_norm.weight", "offset": 1409028096, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 1409042432, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1409056768, "shape": [ 4096, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.11.attn_v.weight", "offset": 1418231808, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.11.ffn_down.weight", "offset": 1422819328, "shape": [ 14336, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.11.ffn_gate.weight", "offset": 1454931968, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.12.attn_norm.weight", "offset": 1487044608, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1487058944, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.12.attn_v.weight", "offset": 1519171584, "shape": [ 3584, 2048 ], "size": 32112640, "type": 3 }, { "name": "blk.12.attn_q.weight", "offset": 1523759104, "shape": [ 3584, 4096 ], "size": 4587520, "type": 3 }, { "name": "blk.12.attn_output.weight", "offset": 1532934144, "shape": [ 4096, 3584 ], "size": 9175040, "type": 3 }, { "name": "blk.12.attn_k.weight", "offset": 1542109184, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.12.ffn_norm.weight", "offset": 1546696704, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 1546711040, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.post_attention_norm.weight", "offset": 1546725376, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 1546739712, "shape": [ 3584, 14336 ], "size": 14336, "type": 3 }, { "name": "blk.12.ffn_gate.weight", "offset": 1578852352, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.13.attn_output.weight", "offset": 1610964992, "shape": [ 4096, 3584 ], "size": 32112640, "type": 3 }, { "name": "blk.13.attn_v.weight", "offset": 1620140032, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.13.ffn_norm.weight", "offset": 1624727552, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 1624741888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.post_attention_norm.weight", "offset": 1624756224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 1624770560, "shape": [ 3584, 14336 ], "size": 14336, "type": 3 }, { "name": "blk.13.ffn_gate.weight", "offset": 1656883200, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.13.ffn_down.weight", "offset": 1688995840, "shape": [ 14336, 3584 ], "size": 32112640, "type": 3 }, { "name": "blk.13.attn_norm.weight", "offset": 1721108480, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1721122816, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.13.attn_q.weight", "offset": 1725710336, "shape": [ 3584, 4096 ], "size": 4587520, "type": 3 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 1734885376, "shape": [ 3584 ], "size": 9175040, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 1734899712, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1734914048, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.14.ffn_gate.weight", "offset": 1767026688, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.14.ffn_up.weight", "offset": 1799139328, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.14.post_attention_norm.weight", "offset": 1831251968, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 1831266304, "shape": [ 4096, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.14.ffn_norm.weight", "offset": 1840441344, "shape": [ 3584 ], "size": 9175040, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1840455680, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.14.attn_v.weight", "offset": 1845043200, "shape": [ 3584, 2048 ], "size": 4587520, "type": 3 }, { "name": "blk.14.attn_q.weight", "offset": 1849630720, "shape": [ 3584, 4096 ], "size": 4587520, "type": 3 }, { "name": "blk.15.attn_v.weight", "offset": 1858805760, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.15.attn_q.weight", "offset": 1863393280, "shape": [ 3584, 4096 ], "size": 4587520, "type": 3 }, { "name": "blk.15.attn_norm.weight", "offset": 1872568320, "shape": [ 3584 ], "size": 9175040, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1872582656, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.15.ffn_gate.weight", "offset": 1904695296, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.15.ffn_up.weight", "offset": 1936807936, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.15.post_attention_norm.weight", "offset": 1968920576, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 1968934912, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 1968949248, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1968963584, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.15.attn_output.weight", "offset": 1973551104, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.16.ffn_norm.weight", "offset": 1982726144, "shape": [ 3584 ], "size": 9175040, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 1982740480, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1982754816, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 1987342336, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1987356672, "shape": [ 3584, 4096 ], "size": 14336, "type": 3 }, { "name": "blk.16.attn_v.weight", "offset": 1996531712, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.16.attn_output.weight", "offset": 2001119232, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.16.post_attention_norm.weight", "offset": 2010294272, "shape": [ 3584 ], "size": 9175040, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 2010308608, "shape": [ 3584, 14336 ], "size": 14336, "type": 3 }, { "name": "blk.16.ffn_gate.weight", "offset": 2042421248, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.16.ffn_down.weight", "offset": 2074533888, "shape": [ 14336, 3584 ], "size": 32112640, "type": 3 }, { "name": "blk.17.attn_norm.weight", "offset": 2106646528, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 2106660864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2106675200, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.17.attn_output.weight", "offset": 2111262720, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 2120437760, "shape": [ 3584 ], "size": 9175040, "type": 0 }, { "name": "blk.17.post_attention_norm.weight", "offset": 2120452096, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 2120466432, "shape": [ 3584, 14336 ], "size": 14336, "type": 3 }, { "name": "blk.17.ffn_gate.weight", "offset": 2152579072, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.17.ffn_down.weight", "offset": 2184691712, "shape": [ 14336, 3584 ], "size": 32112640, "type": 3 }, { "name": "blk.17.attn_v.weight", "offset": 2216804352, "shape": [ 3584, 2048 ], "size": 32112640, "type": 3 }, { "name": "blk.17.attn_q.weight", "offset": 2221391872, "shape": [ 3584, 4096 ], "size": 4587520, "type": 3 }, { "name": "blk.18.attn_v.weight", "offset": 2230566912, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 2235154432, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 2235168768, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2235183104, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.18.ffn_gate.weight", "offset": 2267295744, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.18.ffn_up.weight", "offset": 2299408384, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.18.post_attention_norm.weight", "offset": 2331521024, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2331535360, "shape": [ 3584, 4096 ], "size": 14336, "type": 3 }, { "name": "blk.18.attn_output.weight", "offset": 2340710400, "shape": [ 4096, 3584 ], "size": 9175040, "type": 3 }, { "name": "blk.18.attn_k.weight", "offset": 2349885440, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.18.ffn_norm.weight", "offset": 2354472960, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 2354487296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 2354501632, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2354515968, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.19.ffn_gate.weight", "offset": 2386628608, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.19.ffn_up.weight", "offset": 2418741248, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.19.post_attention_norm.weight", "offset": 2450853888, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 2450868224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2450882560, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.19.attn_output.weight", "offset": 2455470080, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.19.attn_q.weight", "offset": 2464645120, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.19.attn_v.weight", "offset": 2473820160, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.20.attn_k.weight", "offset": 2478407680, "shape": [ 3584, 2048 ], "size": 4587520, "type": 3 }, { "name": "blk.20.attn_output.weight", "offset": 2482995200, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.20.attn_q.weight", "offset": 2492170240, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.20.attn_v.weight", "offset": 2501345280, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.20.attn_norm.weight", "offset": 2505932800, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2505947136, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.20.ffn_gate.weight", "offset": 2538059776, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.20.ffn_up.weight", "offset": 2570172416, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.20.post_attention_norm.weight", "offset": 2602285056, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 2602299392, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 2602313728, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2602328064, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.21.attn_norm.weight", "offset": 2634440704, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.21.ffn_gate.weight", "offset": 2634455040, "shape": [ 3584, 14336 ], "size": 14336, "type": 3 }, { "name": "blk.21.ffn_up.weight", "offset": 2666567680, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.21.post_attention_norm.weight", "offset": 2698680320, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 2698694656, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 2698708992, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2698723328, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.21.attn_output.weight", "offset": 2703310848, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.21.attn_q.weight", "offset": 2712485888, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.21.attn_v.weight", "offset": 2721660928, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.22.attn_norm.weight", "offset": 2726248448, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2726262784, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.22.ffn_gate.weight", "offset": 2758375424, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.22.ffn_up.weight", "offset": 2790488064, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.22.post_attention_norm.weight", "offset": 2822600704, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 2822615040, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 2822629376, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2822643712, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.22.attn_output.weight", "offset": 2827231232, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.22.attn_q.weight", "offset": 2836406272, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.22.attn_v.weight", "offset": 2845581312, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.23.attn_norm.weight", "offset": 2850168832, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2850183168, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.23.ffn_gate.weight", "offset": 2882295808, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.23.ffn_up.weight", "offset": 2914408448, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.23.post_attention_norm.weight", "offset": 2946521088, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 2946535424, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 2946549760, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2946564096, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.23.attn_output.weight", "offset": 2951151616, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.23.attn_q.weight", "offset": 2960326656, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.23.attn_v.weight", "offset": 2969501696, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.24.attn_norm.weight", "offset": 2974089216, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2974103552, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.24.ffn_gate.weight", "offset": 3006216192, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.24.ffn_up.weight", "offset": 3038328832, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.24.post_attention_norm.weight", "offset": 3070441472, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 3070455808, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 3070470144, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 3070484480, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.24.attn_output.weight", "offset": 3075072000, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.24.attn_q.weight", "offset": 3084247040, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.24.attn_v.weight", "offset": 3093422080, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.25.attn_norm.weight", "offset": 3098009600, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3098023936, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.25.ffn_gate.weight", "offset": 3130136576, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.25.ffn_up.weight", "offset": 3162249216, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.25.post_attention_norm.weight", "offset": 3194361856, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 3194376192, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 3194390528, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3194404864, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.25.attn_output.weight", "offset": 3198992384, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.25.attn_q.weight", "offset": 3208167424, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.25.attn_v.weight", "offset": 3217342464, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.26.attn_norm.weight", "offset": 3221929984, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3221944320, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.26.ffn_gate.weight", "offset": 3254056960, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.26.ffn_up.weight", "offset": 3286169600, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.26.post_attention_norm.weight", "offset": 3318282240, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 3318296576, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 3318310912, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3318325248, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.26.attn_output.weight", "offset": 3322912768, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.26.attn_q.weight", "offset": 3332087808, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.26.attn_v.weight", "offset": 3341262848, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.27.attn_norm.weight", "offset": 3345850368, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3345864704, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.27.ffn_gate.weight", "offset": 3377977344, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.27.ffn_up.weight", "offset": 3410089984, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.27.post_attention_norm.weight", "offset": 3442202624, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 3442216960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 3442231296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 3442245632, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.27.attn_output.weight", "offset": 3446833152, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.27.attn_q.weight", "offset": 3456008192, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.27.attn_v.weight", "offset": 3465183232, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.28.attn_norm.weight", "offset": 3469770752, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3469785088, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.28.ffn_gate.weight", "offset": 3501897728, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.28.ffn_up.weight", "offset": 3534010368, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.28.post_attention_norm.weight", "offset": 3566123008, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 3566137344, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 3566151680, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3566166016, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.28.attn_output.weight", "offset": 3570753536, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.28.attn_q.weight", "offset": 3579928576, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.28.attn_v.weight", "offset": 3589103616, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.29.attn_norm.weight", "offset": 3593691136, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3593705472, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.29.ffn_gate.weight", "offset": 3625818112, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.29.ffn_up.weight", "offset": 3657930752, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.29.post_attention_norm.weight", "offset": 3690043392, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 3690057728, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 3690072064, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 3690086400, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.29.attn_output.weight", "offset": 3694673920, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.29.attn_q.weight", "offset": 3703848960, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.29.attn_v.weight", "offset": 3713024000, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.30.attn_norm.weight", "offset": 3717611520, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3717625856, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.30.ffn_gate.weight", "offset": 3749738496, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.30.ffn_up.weight", "offset": 3781851136, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.30.post_attention_norm.weight", "offset": 3813963776, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 3813978112, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 3813992448, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 3814006784, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.30.attn_output.weight", "offset": 3818594304, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.30.attn_q.weight", "offset": 3827769344, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.30.attn_v.weight", "offset": 3836944384, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.31.attn_k.weight", "offset": 3841531904, "shape": [ 3584, 2048 ], "size": 4587520, "type": 3 }, { "name": "blk.31.ffn_down.weight", "offset": 3846119424, "shape": [ 14336, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.31.ffn_gate.weight", "offset": 3878232064, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.31.ffn_up.weight", "offset": 3910344704, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.31.post_attention_norm.weight", "offset": 3942457344, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 3942471680, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 3942486016, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 3942500352, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 3942514688, "shape": [ 4096, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.31.attn_q.weight", "offset": 3951689728, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.31.attn_v.weight", "offset": 3960864768, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.32.ffn_gate.weight", "offset": 3965452288, "shape": [ 3584, 14336 ], "size": 4587520, "type": 3 }, { "name": "blk.32.ffn_up.weight", "offset": 3997564928, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.32.attn_k.weight", "offset": 4029677568, "shape": [ 3584, 2048 ], "size": 32112640, "type": 3 }, { "name": "blk.32.attn_output.weight", "offset": 4034265088, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.32.attn_q.weight", "offset": 4043440128, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.32.attn_v.weight", "offset": 4052615168, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.32.attn_norm.weight", "offset": 4057202688, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 4057217024, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.32.post_attention_norm.weight", "offset": 4089329664, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 4089344000, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 4089358336, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 4089372672, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 4089387008, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.33.ffn_gate.weight", "offset": 4121499648, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.33.ffn_up.weight", "offset": 4153612288, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.33.post_attention_norm.weight", "offset": 4185724928, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 4185739264, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 4185753600, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 4185767936, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.33.attn_output.weight", "offset": 4190355456, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.33.attn_q.weight", "offset": 4199530496, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.33.attn_v.weight", "offset": 4208705536, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.34.attn_norm.weight", "offset": 4213293056, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 4213307392, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.34.ffn_gate.weight", "offset": 4245420032, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.34.ffn_up.weight", "offset": 4277532672, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.34.post_attention_norm.weight", "offset": 4309645312, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 4309659648, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 4309673984, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 4309688320, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.34.attn_output.weight", "offset": 4314275840, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.34.attn_q.weight", "offset": 4323450880, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.34.attn_v.weight", "offset": 4332625920, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.35.attn_norm.weight", "offset": 4337213440, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 4337227776, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.35.ffn_gate.weight", "offset": 4369340416, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.35.ffn_up.weight", "offset": 4401453056, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.35.post_attention_norm.weight", "offset": 4433565696, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 4433580032, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 4433594368, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 4433608704, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.35.attn_output.weight", "offset": 4438196224, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.35.attn_q.weight", "offset": 4447371264, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.35.attn_v.weight", "offset": 4456546304, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.36.ffn_norm.weight", "offset": 4461133824, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 4461148160, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.36.ffn_gate.weight", "offset": 4493260800, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.36.ffn_up.weight", "offset": 4525373440, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.36.post_attention_norm.weight", "offset": 4557486080, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 4557500416, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 4557514752, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 4557529088, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.36.attn_output.weight", "offset": 4562116608, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.36.attn_q.weight", "offset": 4571291648, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.36.attn_v.weight", "offset": 4580466688, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.37.attn_norm.weight", "offset": 4585054208, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 4585068544, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.37.ffn_gate.weight", "offset": 4617181184, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.37.ffn_up.weight", "offset": 4649293824, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.37.post_attention_norm.weight", "offset": 4681406464, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 4681420800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 4681435136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 4681449472, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.37.attn_output.weight", "offset": 4686036992, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.37.attn_q.weight", "offset": 4695212032, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.37.attn_v.weight", "offset": 4704387072, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.38.attn_norm.weight", "offset": 4708974592, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 4708988928, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.38.ffn_gate.weight", "offset": 4741101568, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.38.ffn_up.weight", "offset": 4773214208, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.38.post_attention_norm.weight", "offset": 4805326848, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 4805341184, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 4805355520, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 4805369856, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.38.attn_output.weight", "offset": 4809957376, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.38.attn_q.weight", "offset": 4819132416, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.38.attn_v.weight", "offset": 4828307456, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.39.attn_norm.weight", "offset": 4832894976, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 4832909312, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.39.ffn_gate.weight", "offset": 4865021952, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.39.ffn_up.weight", "offset": 4897134592, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.39.post_attention_norm.weight", "offset": 4929247232, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 4929261568, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 4929275904, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 4929290240, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.39.attn_output.weight", "offset": 4933877760, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.39.attn_q.weight", "offset": 4943052800, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.39.attn_v.weight", "offset": 4952227840, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.40.attn_norm.weight", "offset": 4956815360, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 4956829696, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.40.ffn_gate.weight", "offset": 4988942336, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.40.ffn_up.weight", "offset": 5021054976, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.40.post_attention_norm.weight", "offset": 5053167616, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 5053181952, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 5053196288, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 5053210624, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.40.attn_output.weight", "offset": 5057798144, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.40.attn_q.weight", "offset": 5066973184, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.40.attn_v.weight", "offset": 5076148224, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "blk.41.attn_norm.weight", "offset": 5080735744, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 5080750080, "shape": [ 14336, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.41.ffn_gate.weight", "offset": 5112862720, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.41.ffn_up.weight", "offset": 5144975360, "shape": [ 3584, 14336 ], "size": 32112640, "type": 3 }, { "name": "blk.41.post_attention_norm.weight", "offset": 5177088000, "shape": [ 3584 ], "size": 32112640, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 5177102336, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 5177116672, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 5177131008, "shape": [ 3584, 2048 ], "size": 14336, "type": 3 }, { "name": "blk.41.attn_output.weight", "offset": 5181718528, "shape": [ 4096, 3584 ], "size": 4587520, "type": 3 }, { "name": "blk.41.attn_q.weight", "offset": 5190893568, "shape": [ 3584, 4096 ], "size": 9175040, "type": 3 }, { "name": "blk.41.attn_v.weight", "offset": 5200068608, "shape": [ 3584, 2048 ], "size": 9175040, "type": 3 }, { "name": "output_norm.weight", "offset": 5204656128, "shape": [ 3584 ], "size": 4587520, "type": 0 }, { "name": "token_embd.weight", "offset": 5204670464, "shape": [ 3584, 256000 ], "size": 14336, "type": 14 } ], "version": 3 }