gemma2

{ "metadata": { "gemma2.attention.head_count": 32, "gemma2.attention.head_count_kv": 16, "gemma2.attention.key_length": 128, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 128, "gemma2.block_count": 46, "gemma2.context_length": 8192, "gemma2.embedding_length": 4608, "gemma2.feed_forward_length": 36864, "general.architecture": "gemma2", "general.file_type": 10, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 27227128320, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 18432, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 73009152, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.0.ffn_up.weight", "offset": 128747520, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.0.post_attention_norm.weight", "offset": 184485888, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 184504320, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 184522752, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 184541184, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.0.attn_output.weight", "offset": 187637760, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.0.attn_q.weight", "offset": 195747840, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.0.attn_v.weight", "offset": 201940992, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 205996032, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 206014464, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.1.ffn_gate.weight", "offset": 279005184, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.1.ffn_up.weight", "offset": 334743552, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.1.post_attention_norm.weight", "offset": 390481920, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 390500352, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 390518784, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 390537216, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.1.attn_output.weight", "offset": 393633792, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.1.attn_q.weight", "offset": 401743872, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.1.attn_v.weight", "offset": 407937024, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.2.attn_k.weight", "offset": 411992064, "shape": [ 4608, 2048 ], "size": 4055040, "type": 10 }, { "name": "blk.2.attn_output.weight", "offset": 415088640, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.2.attn_q.weight", "offset": 423198720, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.2.attn_v.weight", "offset": 429391872, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 433446912, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 433465344, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.2.ffn_gate.weight", "offset": 506456064, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.2.ffn_up.weight", "offset": 562194432, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.2.post_attention_norm.weight", "offset": 617932800, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 617951232, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 617969664, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_norm.weight", "offset": 617988096, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 618006528, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.3.ffn_gate.weight", "offset": 690997248, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.3.ffn_up.weight", "offset": 746735616, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.3.post_attention_norm.weight", "offset": 802473984, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 802492416, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 802510848, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 802529280, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.3.attn_output.weight", "offset": 805625856, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.3.attn_q.weight", "offset": 813735936, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.3.attn_v.weight", "offset": 819929088, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 823984128, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 824002560, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.4.ffn_gate.weight", "offset": 896993280, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.4.ffn_up.weight", "offset": 952731648, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.4.post_attention_norm.weight", "offset": 1008470016, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 1008488448, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 1008506880, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1008525312, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.4.attn_output.weight", "offset": 1011621888, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.4.attn_q.weight", "offset": 1019731968, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.4.attn_v.weight", "offset": 1025925120, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 1029980160, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1029998592, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.5.ffn_gate.weight", "offset": 1102989312, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.5.ffn_up.weight", "offset": 1158727680, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.5.post_attention_norm.weight", "offset": 1214466048, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 1214484480, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 1214502912, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1214521344, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.5.attn_output.weight", "offset": 1217617920, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.5.attn_q.weight", "offset": 1225728000, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.5.attn_v.weight", "offset": 1231921152, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.6.ffn_gate.weight", "offset": 1235976192, "shape": [ 4608, 36864 ], "size": 4055040, "type": 10 }, { "name": "blk.6.attn_k.weight", "offset": 1291714560, "shape": [ 4608, 2048 ], "size": 55738368, "type": 10 }, { "name": "blk.6.attn_output.weight", "offset": 1294811136, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.6.attn_q.weight", "offset": 1302921216, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.6.attn_v.weight", "offset": 1309114368, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 1313169408, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1313187840, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 1386178560, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.6.post_attention_norm.weight", "offset": 1441916928, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 1441935360, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 1441953792, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1441972224, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1441990656, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.7.ffn_gate.weight", "offset": 1514981376, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.7.ffn_up.weight", "offset": 1570719744, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.7.post_attention_norm.weight", "offset": 1626458112, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 1626476544, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 1626494976, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1626513408, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.7.attn_output.weight", "offset": 1629609984, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.7.attn_q.weight", "offset": 1637720064, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.7.attn_v.weight", "offset": 1643913216, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 1647968256, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1647986688, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.8.ffn_gate.weight", "offset": 1720977408, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.8.ffn_up.weight", "offset": 1776715776, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.8.post_attention_norm.weight", "offset": 1832454144, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 1832472576, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 1832491008, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1832509440, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.8.attn_output.weight", "offset": 1835606016, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.8.attn_q.weight", "offset": 1843716096, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.8.attn_v.weight", "offset": 1849909248, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 1853964288, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1853982720, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.9.ffn_gate.weight", "offset": 1926973440, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.9.ffn_up.weight", "offset": 1982711808, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.9.post_attention_norm.weight", "offset": 2038450176, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 2038468608, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 2038487040, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2038505472, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.9.attn_output.weight", "offset": 2041602048, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.9.attn_q.weight", "offset": 2049712128, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.9.attn_v.weight", "offset": 2055905280, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.10.ffn_gate.weight", "offset": 2059960320, "shape": [ 4608, 36864 ], "size": 4055040, "type": 10 }, { "name": "blk.10.ffn_up.weight", "offset": 2115698688, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.10.attn_k.weight", "offset": 2171437056, "shape": [ 4608, 2048 ], "size": 55738368, "type": 10 }, { "name": "blk.10.attn_output.weight", "offset": 2174533632, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.10.attn_q.weight", "offset": 2182643712, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.10.attn_v.weight", "offset": 2188836864, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.10.attn_norm.weight", "offset": 2192891904, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 2192910336, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.10.post_attention_norm.weight", "offset": 2265901056, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 2265919488, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 2265937920, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_norm.weight", "offset": 2265956352, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 2265974784, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.11.ffn_gate.weight", "offset": 2338965504, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.11.ffn_up.weight", "offset": 2394703872, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.11.post_attention_norm.weight", "offset": 2450442240, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 2450460672, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 2450479104, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 2450497536, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.11.attn_output.weight", "offset": 2453594112, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.11.attn_q.weight", "offset": 2461704192, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.11.attn_v.weight", "offset": 2467897344, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 2471952384, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2471970816, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 2544961536, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.12.ffn_up.weight", "offset": 2600699904, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.12.post_attention_norm.weight", "offset": 2656438272, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 2656456704, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 2656475136, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 2656493568, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.12.attn_output.weight", "offset": 2659590144, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.12.attn_q.weight", "offset": 2667700224, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.12.attn_v.weight", "offset": 2673893376, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 2677948416, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2677966848, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 2750957568, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.13.ffn_up.weight", "offset": 2806695936, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.13.post_attention_norm.weight", "offset": 2862434304, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 2862452736, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 2862471168, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2862489600, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.13.attn_output.weight", "offset": 2865586176, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.13.attn_q.weight", "offset": 2873696256, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.13.attn_v.weight", "offset": 2879889408, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 2883944448, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2883962880, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.14.ffn_gate.weight", "offset": 2956953600, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.14.ffn_up.weight", "offset": 3012691968, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.14.post_attention_norm.weight", "offset": 3068430336, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 3068448768, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 3068467200, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 3068485632, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.14.attn_output.weight", "offset": 3071582208, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.14.attn_q.weight", "offset": 3079692288, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.14.attn_v.weight", "offset": 3085885440, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.15.attn_k.weight", "offset": 3089940480, "shape": [ 4608, 2048 ], "size": 4055040, "type": 10 }, { "name": "blk.15.attn_output.weight", "offset": 3093037056, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.15.attn_q.weight", "offset": 3101147136, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.15.attn_v.weight", "offset": 3107340288, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 3111395328, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3111413760, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.15.ffn_gate.weight", "offset": 3184404480, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.15.ffn_up.weight", "offset": 3240142848, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.15.post_attention_norm.weight", "offset": 3295881216, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 3295899648, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 3295918080, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 3295936512, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3295954944, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 3368945664, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.16.ffn_up.weight", "offset": 3424684032, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.16.post_attention_norm.weight", "offset": 3480422400, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 3480440832, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 3480459264, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 3480477696, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.16.attn_output.weight", "offset": 3483574272, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.16.attn_q.weight", "offset": 3491684352, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.16.attn_v.weight", "offset": 3497877504, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 3501932544, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3501950976, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 3574941696, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.17.ffn_up.weight", "offset": 3630680064, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.17.post_attention_norm.weight", "offset": 3686418432, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 3686436864, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 3686455296, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 3686473728, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.17.attn_output.weight", "offset": 3689570304, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.17.attn_q.weight", "offset": 3697680384, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.17.attn_v.weight", "offset": 3703873536, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 3707928576, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3707947008, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.18.ffn_gate.weight", "offset": 3780937728, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.18.ffn_up.weight", "offset": 3836676096, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.18.post_attention_norm.weight", "offset": 3892414464, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 3892432896, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 3892451328, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 3892469760, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.18.attn_output.weight", "offset": 3895566336, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.18.attn_q.weight", "offset": 3903676416, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.18.attn_v.weight", "offset": 3909869568, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.19.ffn_gate.weight", "offset": 3913924608, "shape": [ 4608, 36864 ], "size": 4055040, "type": 10 }, { "name": "blk.19.attn_k.weight", "offset": 3969662976, "shape": [ 4608, 2048 ], "size": 55738368, "type": 10 }, { "name": "blk.19.attn_output.weight", "offset": 3972759552, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.19.attn_q.weight", "offset": 3980869632, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.19.attn_v.weight", "offset": 3987062784, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 3991117824, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3991136256, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 4064126976, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.19.post_attention_norm.weight", "offset": 4119865344, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 4119883776, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 4119902208, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_norm.weight", "offset": 4119920640, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4119939072, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.20.ffn_gate.weight", "offset": 4192929792, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.20.ffn_up.weight", "offset": 4248668160, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.20.post_attention_norm.weight", "offset": 4304406528, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 4304424960, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 4304443392, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 4304461824, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.20.attn_output.weight", "offset": 4307558400, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.20.attn_q.weight", "offset": 4315668480, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.20.attn_v.weight", "offset": 4321861632, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 4325916672, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4325935104, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.21.ffn_gate.weight", "offset": 4398925824, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.21.ffn_up.weight", "offset": 4454664192, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.21.post_attention_norm.weight", "offset": 4510402560, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 4510420992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 4510439424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 4510457856, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.21.attn_output.weight", "offset": 4513554432, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.21.attn_q.weight", "offset": 4521664512, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.21.attn_v.weight", "offset": 4527857664, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 4531912704, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4531931136, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.22.ffn_gate.weight", "offset": 4604921856, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.22.ffn_up.weight", "offset": 4660660224, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.22.post_attention_norm.weight", "offset": 4716398592, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 4716417024, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 4716435456, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 4716453888, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.22.attn_output.weight", "offset": 4719550464, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.22.attn_q.weight", "offset": 4727660544, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.22.attn_v.weight", "offset": 4733853696, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 4737908736, "shape": [ 4608, 36864 ], "size": 4055040, "type": 10 }, { "name": "blk.23.ffn_gate.weight", "offset": 4793647104, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.23.attn_k.weight", "offset": 4849385472, "shape": [ 4608, 2048 ], "size": 55738368, "type": 10 }, { "name": "blk.23.attn_output.weight", "offset": 4852482048, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.23.attn_q.weight", "offset": 4860592128, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.23.attn_v.weight", "offset": 4866785280, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.23.attn_norm.weight", "offset": 4870840320, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4870858752, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.23.post_attention_norm.weight", "offset": 4943849472, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 4943867904, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 4943886336, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 4943904768, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4943923200, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.24.ffn_gate.weight", "offset": 5016913920, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.24.ffn_up.weight", "offset": 5072652288, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.24.post_attention_norm.weight", "offset": 5128390656, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 5128409088, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 5128427520, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 5128445952, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.24.attn_output.weight", "offset": 5131542528, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.24.attn_q.weight", "offset": 5139652608, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.24.attn_v.weight", "offset": 5145845760, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 5149900800, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5149919232, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.25.ffn_gate.weight", "offset": 5222909952, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.25.ffn_up.weight", "offset": 5278648320, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.25.post_attention_norm.weight", "offset": 5334386688, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 5334405120, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 5334423552, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 5334441984, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.25.attn_output.weight", "offset": 5337538560, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.25.attn_q.weight", "offset": 5345648640, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.25.attn_v.weight", "offset": 5351841792, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 5355896832, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5355915264, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.26.ffn_gate.weight", "offset": 5428905984, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.26.ffn_up.weight", "offset": 5484644352, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.26.post_attention_norm.weight", "offset": 5540382720, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 5540401152, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 5540419584, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 5540438016, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.26.attn_output.weight", "offset": 5543534592, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.26.attn_q.weight", "offset": 5551644672, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.26.attn_v.weight", "offset": 5557837824, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 5561892864, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5561911296, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.27.ffn_gate.weight", "offset": 5634902016, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.27.ffn_up.weight", "offset": 5690640384, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.27.post_attention_norm.weight", "offset": 5746378752, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 5746397184, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 5746415616, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 5746434048, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.27.attn_output.weight", "offset": 5749530624, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.27.attn_q.weight", "offset": 5757640704, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.27.attn_v.weight", "offset": 5763833856, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.28.attn_k.weight", "offset": 5767888896, "shape": [ 4608, 2048 ], "size": 4055040, "type": 10 }, { "name": "blk.28.attn_output.weight", "offset": 5770985472, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.28.attn_q.weight", "offset": 5779095552, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.28.attn_v.weight", "offset": 5785288704, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 5789343744, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5789362176, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.28.ffn_gate.weight", "offset": 5862352896, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.28.ffn_up.weight", "offset": 5918091264, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.28.post_attention_norm.weight", "offset": 5973829632, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 5973848064, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 5973866496, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 5973884928, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 5973903360, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.29.ffn_gate.weight", "offset": 6046894080, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.29.ffn_up.weight", "offset": 6102632448, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.29.post_attention_norm.weight", "offset": 6158370816, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 6158389248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 6158407680, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 6158426112, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.29.attn_output.weight", "offset": 6161522688, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.29.attn_q.weight", "offset": 6169632768, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.29.attn_v.weight", "offset": 6175825920, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.30.attn_norm.weight", "offset": 6179880960, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 6179899392, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.30.ffn_gate.weight", "offset": 6252890112, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.30.ffn_up.weight", "offset": 6308628480, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.30.post_attention_norm.weight", "offset": 6364366848, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 6364385280, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 6364403712, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 6364422144, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.30.attn_output.weight", "offset": 6367518720, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.30.attn_q.weight", "offset": 6375628800, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.30.attn_v.weight", "offset": 6381821952, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.31.attn_norm.weight", "offset": 6385876992, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 6385895424, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.31.ffn_gate.weight", "offset": 6458886144, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.31.ffn_up.weight", "offset": 6514624512, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.31.post_attention_norm.weight", "offset": 6570362880, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 6570381312, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 6570399744, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 6570418176, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.31.attn_output.weight", "offset": 6573514752, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.31.attn_q.weight", "offset": 6581624832, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.31.attn_v.weight", "offset": 6587817984, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.32.ffn_gate.weight", "offset": 6591873024, "shape": [ 4608, 36864 ], "size": 4055040, "type": 10 }, { "name": "blk.32.attn_k.weight", "offset": 6647611392, "shape": [ 4608, 2048 ], "size": 55738368, "type": 10 }, { "name": "blk.32.attn_output.weight", "offset": 6650707968, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.32.attn_q.weight", "offset": 6658818048, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.32.attn_v.weight", "offset": 6665011200, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.32.attn_norm.weight", "offset": 6669066240, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 6669084672, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 6742075392, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.32.post_attention_norm.weight", "offset": 6797813760, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 6797832192, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 6797850624, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 6797869056, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6797887488, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.33.ffn_gate.weight", "offset": 6870878208, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.33.ffn_up.weight", "offset": 6926616576, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.33.post_attention_norm.weight", "offset": 6982354944, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 6982373376, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 6982391808, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 6982410240, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.33.attn_output.weight", "offset": 6985506816, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.33.attn_q.weight", "offset": 6993616896, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.33.attn_v.weight", "offset": 6999810048, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 7003865088, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 7003883520, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.34.ffn_gate.weight", "offset": 7076874240, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.34.ffn_up.weight", "offset": 7132612608, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.34.post_attention_norm.weight", "offset": 7188350976, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 7188369408, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_norm.weight", "offset": 7188387840, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 7188406272, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.34.attn_output.weight", "offset": 7191502848, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.34.attn_q.weight", "offset": 7199612928, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.34.attn_v.weight", "offset": 7205806080, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.35.attn_norm.weight", "offset": 7209861120, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 7209879552, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.35.ffn_gate.weight", "offset": 7282870272, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.35.ffn_up.weight", "offset": 7338608640, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.35.post_attention_norm.weight", "offset": 7394347008, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 7394365440, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 7394383872, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 7394402304, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.35.attn_output.weight", "offset": 7397498880, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.35.attn_q.weight", "offset": 7405608960, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.35.attn_v.weight", "offset": 7411802112, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.36.ffn_gate.weight", "offset": 7415857152, "shape": [ 4608, 36864 ], "size": 4055040, "type": 10 }, { "name": "blk.36.ffn_up.weight", "offset": 7471595520, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.36.attn_k.weight", "offset": 7527333888, "shape": [ 4608, 2048 ], "size": 55738368, "type": 10 }, { "name": "blk.36.attn_output.weight", "offset": 7530430464, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.36.attn_q.weight", "offset": 7538540544, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.36.attn_v.weight", "offset": 7544733696, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.36.attn_norm.weight", "offset": 7548788736, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 7548807168, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.36.post_attention_norm.weight", "offset": 7621797888, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 7621816320, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 7621834752, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 7621853184, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 7621871616, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.37.ffn_gate.weight", "offset": 7694862336, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.37.ffn_up.weight", "offset": 7750600704, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.37.post_attention_norm.weight", "offset": 7806339072, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 7806357504, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 7806375936, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 7806394368, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.37.attn_output.weight", "offset": 7809490944, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.37.attn_q.weight", "offset": 7817601024, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.37.attn_v.weight", "offset": 7823794176, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.38.attn_norm.weight", "offset": 7827849216, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 7827867648, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.38.ffn_gate.weight", "offset": 7900858368, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.38.ffn_up.weight", "offset": 7956596736, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.38.post_attention_norm.weight", "offset": 8012335104, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 8012353536, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 8012371968, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 8012390400, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.38.attn_output.weight", "offset": 8015486976, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.38.attn_q.weight", "offset": 8023597056, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.38.attn_v.weight", "offset": 8029790208, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.39.attn_norm.weight", "offset": 8033845248, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 8033863680, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.39.ffn_gate.weight", "offset": 8106854400, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.39.ffn_up.weight", "offset": 8162592768, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.39.post_attention_norm.weight", "offset": 8218331136, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 8218349568, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 8218368000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 8218386432, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.39.attn_output.weight", "offset": 8221483008, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.39.attn_q.weight", "offset": 8229593088, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.39.attn_v.weight", "offset": 8235786240, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.40.ffn_down.weight", "offset": 8239841280, "shape": [ 36864, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.40.attn_norm.weight", "offset": 8312832000, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.40.ffn_gate.weight", "offset": 8312850432, "shape": [ 4608, 36864 ], "size": 18432, "type": 10 }, { "name": "blk.40.ffn_up.weight", "offset": 8368588800, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.40.post_attention_norm.weight", "offset": 8424327168, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 8424345600, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 8424364032, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 8424382464, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.40.attn_output.weight", "offset": 8427479040, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.40.attn_q.weight", "offset": 8435589120, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.40.attn_v.weight", "offset": 8441782272, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.41.attn_k.weight", "offset": 8445837312, "shape": [ 4608, 2048 ], "size": 4055040, "type": 10 }, { "name": "blk.41.attn_output.weight", "offset": 8448933888, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.41.attn_q.weight", "offset": 8457043968, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.41.attn_v.weight", "offset": 8463237120, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.41.attn_norm.weight", "offset": 8467292160, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 8467310592, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.41.ffn_gate.weight", "offset": 8540301312, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.41.ffn_up.weight", "offset": 8596039680, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.41.post_attention_norm.weight", "offset": 8651778048, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 8651796480, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 8651814912, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 8651833344, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 8651851776, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.42.ffn_gate.weight", "offset": 8724842496, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.42.ffn_up.weight", "offset": 8780580864, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.42.post_attention_norm.weight", "offset": 8836319232, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.42.post_ffw_norm.weight", "offset": 8836337664, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 8836356096, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 8836374528, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.42.attn_output.weight", "offset": 8839471104, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.42.attn_q.weight", "offset": 8847581184, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.42.attn_v.weight", "offset": 8853774336, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.43.attn_norm.weight", "offset": 8857829376, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 8857847808, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.43.ffn_gate.weight", "offset": 8930838528, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.43.ffn_up.weight", "offset": 8986576896, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.43.post_attention_norm.weight", "offset": 9042315264, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.43.post_ffw_norm.weight", "offset": 9042333696, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 9042352128, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 9042370560, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.43.attn_output.weight", "offset": 9045467136, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.43.attn_q.weight", "offset": 9053577216, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.43.attn_v.weight", "offset": 9059770368, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.44.attn_norm.weight", "offset": 9063825408, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 9063843840, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.44.ffn_gate.weight", "offset": 9136834560, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.44.ffn_up.weight", "offset": 9192572928, "shape": [ 4608, 36864 ], "size": 55738368, "type": 10 }, { "name": "blk.44.post_attention_norm.weight", "offset": 9248311296, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.44.post_ffw_norm.weight", "offset": 9248329728, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 9248348160, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 9248366592, "shape": [ 4608, 2048 ], "size": 18432, "type": 10 }, { "name": "blk.44.attn_output.weight", "offset": 9251463168, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.44.attn_q.weight", "offset": 9259573248, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.44.attn_v.weight", "offset": 9265766400, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.45.ffn_gate.weight", "offset": 9269821440, "shape": [ 4608, 36864 ], "size": 4055040, "type": 10 }, { "name": "blk.45.attn_k.weight", "offset": 9325559808, "shape": [ 4608, 2048 ], "size": 55738368, "type": 10 }, { "name": "blk.45.attn_output.weight", "offset": 9328656384, "shape": [ 4096, 4608 ], "size": 3096576, "type": 11 }, { "name": "blk.45.attn_q.weight", "offset": 9336766464, "shape": [ 4608, 4096 ], "size": 8110080, "type": 10 }, { "name": "blk.45.attn_v.weight", "offset": 9342959616, "shape": [ 4608, 2048 ], "size": 6193152, "type": 11 }, { "name": "blk.45.attn_norm.weight", "offset": 9347014656, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 9347033088, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.45.ffn_up.weight", "offset": 9420023808, "shape": [ 4608, 36864 ], "size": 72990720, "type": 10 }, { "name": "blk.45.post_attention_norm.weight", "offset": 9475762176, "shape": [ 4608 ], "size": 55738368, "type": 0 }, { "name": "blk.45.post_ffw_norm.weight", "offset": 9475780608, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 9475799040, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "output_norm.weight", "offset": 9475817472, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "token_embd.weight", "offset": 9475835904, "shape": [ 4608, 256000 ], "size": 18432, "type": 14 } ], "version": 3 }