gemma2

{ "metadata": { "gemma2.attention.head_count": 16, "gemma2.attention.head_count_kv": 8, "gemma2.attention.key_length": 256, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 256, "gemma2.block_count": 42, "gemma2.context_length": 8192, "gemma2.embedding_length": 3584, "gemma2.feed_forward_length": 14336, "general.architecture": "gemma2", "general.file_type": 1, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 9241705984, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 14336, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.0.ffn_gate.weight", "offset": 102774784, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.0.ffn_up.weight", "offset": 205535232, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.0.post_attention_norm.weight", "offset": 308295680, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 308310016, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 308324352, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 308338688, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.0.attn_output.weight", "offset": 323018752, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.0.attn_q.weight", "offset": 352378880, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.0.attn_v.weight", "offset": 381739008, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.1.attn_norm.weight", "offset": 396419072, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 396433408, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.1.ffn_gate.weight", "offset": 499193856, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.1.ffn_up.weight", "offset": 601954304, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.1.post_attention_norm.weight", "offset": 704714752, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 704729088, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 704743424, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 704757760, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.1.attn_output.weight", "offset": 719437824, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.1.attn_q.weight", "offset": 748797952, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.1.attn_v.weight", "offset": 778158080, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.2.attn_v.weight", "offset": 792838144, "shape": [ 3584, 2048 ], "size": 14680064, "type": 1 }, { "name": "blk.2.ffn_down.weight", "offset": 807518208, "shape": [ 14336, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.2.ffn_gate.weight", "offset": 910278656, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.2.ffn_up.weight", "offset": 1013039104, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.2.post_attention_norm.weight", "offset": 1115799552, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 1115813888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 1115828224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1115842560, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.2.attn_output.weight", "offset": 1130522624, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.2.attn_q.weight", "offset": 1159882752, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 1189242880, "shape": [ 3584 ], "size": 29360128, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 1189257216, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.3.ffn_down.weight", "offset": 1203937280, "shape": [ 14336, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.3.ffn_gate.weight", "offset": 1306697728, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.3.ffn_up.weight", "offset": 1409458176, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.3.post_attention_norm.weight", "offset": 1512218624, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 1512232960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 1512247296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1512261632, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.3.attn_output.weight", "offset": 1526941696, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.3.attn_q.weight", "offset": 1556301824, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.3.attn_norm.weight", "offset": 1585661952, "shape": [ 3584 ], "size": 29360128, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 1585676288, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1585690624, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.4.ffn_gate.weight", "offset": 1688451072, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.4.ffn_up.weight", "offset": 1791211520, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.4.post_attention_norm.weight", "offset": 1893971968, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 1893986304, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 1894000640, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1894014976, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.4.attn_output.weight", "offset": 1908695040, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.4.attn_q.weight", "offset": 1938055168, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.4.attn_v.weight", "offset": 1967415296, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.5.attn_norm.weight", "offset": 1982095360, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1982109696, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.5.ffn_gate.weight", "offset": 2084870144, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.5.ffn_up.weight", "offset": 2187630592, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.5.post_attention_norm.weight", "offset": 2290391040, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 2290405376, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 2290419712, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 2290434048, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.5.attn_output.weight", "offset": 2305114112, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.5.attn_q.weight", "offset": 2334474240, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.5.attn_v.weight", "offset": 2363834368, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.6.attn_norm.weight", "offset": 2378514432, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2378528768, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.6.ffn_gate.weight", "offset": 2481289216, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.6.ffn_up.weight", "offset": 2584049664, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.6.post_attention_norm.weight", "offset": 2686810112, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 2686824448, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 2686838784, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 2686853120, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.6.attn_output.weight", "offset": 2701533184, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.6.attn_q.weight", "offset": 2730893312, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.6.attn_v.weight", "offset": 2760253440, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.7.attn_norm.weight", "offset": 2774933504, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 2774947840, "shape": [ 3584, 14336 ], "size": 14336, "type": 1 }, { "name": "blk.7.attn_k.weight", "offset": 2877708288, "shape": [ 3584, 2048 ], "size": 102760448, "type": 1 }, { "name": "blk.7.attn_output.weight", "offset": 2892388352, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.7.attn_q.weight", "offset": 2921748480, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.7.attn_v.weight", "offset": 2951108608, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.7.ffn_gate.weight", "offset": 2965788672, "shape": [ 3584, 14336 ], "size": 14680064, "type": 1 }, { "name": "blk.7.ffn_down.weight", "offset": 3068549120, "shape": [ 14336, 3584 ], "size": 102760448, "type": 1 }, { "name": "blk.7.post_attention_norm.weight", "offset": 3171309568, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 3171323904, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 3171338240, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 3171352576, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3171366912, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.8.ffn_gate.weight", "offset": 3274127360, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.8.ffn_up.weight", "offset": 3376887808, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 3479648256, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 3479662592, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 3479676928, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.8.attn_output.weight", "offset": 3494356992, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.8.attn_q.weight", "offset": 3523717120, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.8.attn_v.weight", "offset": 3553077248, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.8.post_attention_norm.weight", "offset": 3567757312, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 3567771648, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.9.ffn_down.weight", "offset": 3582451712, "shape": [ 14336, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 3685212160, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.9.ffn_gate.weight", "offset": 3685226496, "shape": [ 3584, 14336 ], "size": 14336, "type": 1 }, { "name": "blk.9.ffn_up.weight", "offset": 3787986944, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.9.post_attention_norm.weight", "offset": 3890747392, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 3890761728, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 3890776064, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 3890790400, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.9.attn_output.weight", "offset": 3905470464, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.9.attn_q.weight", "offset": 3934830592, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.10.attn_q.weight", "offset": 3964190720, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.10.attn_v.weight", "offset": 3993550848, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.10.ffn_down.weight", "offset": 4008230912, "shape": [ 14336, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.10.ffn_gate.weight", "offset": 4110991360, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.10.attn_k.weight", "offset": 4213751808, "shape": [ 3584, 2048 ], "size": 102760448, "type": 1 }, { "name": "blk.10.ffn_norm.weight", "offset": 4228431872, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 4228446208, "shape": [ 3584, 14336 ], "size": 14336, "type": 1 }, { "name": "blk.10.post_attention_norm.weight", "offset": 4331206656, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 4331220992, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 4331235328, "shape": [ 4096, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.10.attn_norm.weight", "offset": 4360595456, "shape": [ 3584 ], "size": 29360128, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 4360609792, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.post_attention_norm.weight", "offset": 4360624128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 4360638464, "shape": [ 3584, 4096 ], "size": 14336, "type": 1 }, { "name": "blk.11.ffn_up.weight", "offset": 4389998592, "shape": [ 3584, 14336 ], "size": 29360128, "type": 1 }, { "name": "blk.11.attn_k.weight", "offset": 4492759040, "shape": [ 3584, 2048 ], "size": 102760448, "type": 1 }, { "name": "blk.11.attn_norm.weight", "offset": 4507439104, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 4507453440, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 4507467776, "shape": [ 4096, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.11.attn_v.weight", "offset": 4536827904, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.11.ffn_down.weight", "offset": 4551507968, "shape": [ 14336, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.11.ffn_gate.weight", "offset": 4654268416, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 4757028864, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 4757043200, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.12.attn_v.weight", "offset": 4859803648, "shape": [ 3584, 2048 ], "size": 102760448, "type": 1 }, { "name": "blk.12.attn_q.weight", "offset": 4874483712, "shape": [ 3584, 4096 ], "size": 14680064, "type": 1 }, { "name": "blk.12.attn_output.weight", "offset": 4903843840, "shape": [ 4096, 3584 ], "size": 29360128, "type": 1 }, { "name": "blk.12.attn_k.weight", "offset": 4933203968, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.12.ffn_norm.weight", "offset": 4947884032, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 4947898368, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.post_attention_norm.weight", "offset": 4947912704, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 4947927040, "shape": [ 3584, 14336 ], "size": 14336, "type": 1 }, { "name": "blk.12.ffn_gate.weight", "offset": 5050687488, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.13.attn_output.weight", "offset": 5153447936, "shape": [ 4096, 3584 ], "size": 102760448, "type": 1 }, { "name": "blk.13.attn_v.weight", "offset": 5182808064, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.13.ffn_norm.weight", "offset": 5197488128, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 5197502464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.post_attention_norm.weight", "offset": 5197516800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 5197531136, "shape": [ 3584, 14336 ], "size": 14336, "type": 1 }, { "name": "blk.13.ffn_gate.weight", "offset": 5300291584, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.13.ffn_down.weight", "offset": 5403052032, "shape": [ 14336, 3584 ], "size": 102760448, "type": 1 }, { "name": "blk.13.attn_norm.weight", "offset": 5505812480, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 5505826816, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.13.attn_q.weight", "offset": 5520506880, "shape": [ 3584, 4096 ], "size": 14680064, "type": 1 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 5549867008, "shape": [ 3584 ], "size": 29360128, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 5549881344, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 5549895680, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.14.ffn_gate.weight", "offset": 5652656128, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.14.ffn_up.weight", "offset": 5755416576, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.14.post_attention_norm.weight", "offset": 5858177024, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 5858191360, "shape": [ 4096, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.14.ffn_norm.weight", "offset": 5887551488, "shape": [ 3584 ], "size": 29360128, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 5887565824, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.14.attn_v.weight", "offset": 5902245888, "shape": [ 3584, 2048 ], "size": 14680064, "type": 1 }, { "name": "blk.14.attn_q.weight", "offset": 5916925952, "shape": [ 3584, 4096 ], "size": 14680064, "type": 1 }, { "name": "blk.15.attn_v.weight", "offset": 5946286080, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.15.attn_q.weight", "offset": 5960966144, "shape": [ 3584, 4096 ], "size": 14680064, "type": 1 }, { "name": "blk.15.attn_norm.weight", "offset": 5990326272, "shape": [ 3584 ], "size": 29360128, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 5990340608, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.15.ffn_gate.weight", "offset": 6093101056, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.15.ffn_up.weight", "offset": 6195861504, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.15.post_attention_norm.weight", "offset": 6298621952, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 6298636288, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 6298650624, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 6298664960, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.15.attn_output.weight", "offset": 6313345024, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.16.ffn_norm.weight", "offset": 6342705152, "shape": [ 3584 ], "size": 29360128, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 6342719488, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 6342733824, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 6357413888, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 6357428224, "shape": [ 3584, 4096 ], "size": 14336, "type": 1 }, { "name": "blk.16.attn_v.weight", "offset": 6386788352, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.16.attn_output.weight", "offset": 6401468416, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.16.post_attention_norm.weight", "offset": 6430828544, "shape": [ 3584 ], "size": 29360128, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 6430842880, "shape": [ 3584, 14336 ], "size": 14336, "type": 1 }, { "name": "blk.16.ffn_gate.weight", "offset": 6533603328, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.16.ffn_down.weight", "offset": 6636363776, "shape": [ 14336, 3584 ], "size": 102760448, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 6739124224, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 6739138560, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 6739152896, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.17.attn_output.weight", "offset": 6753832960, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 6783193088, "shape": [ 3584 ], "size": 29360128, "type": 0 }, { "name": "blk.17.post_attention_norm.weight", "offset": 6783207424, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 6783221760, "shape": [ 3584, 14336 ], "size": 14336, "type": 1 }, { "name": "blk.17.ffn_gate.weight", "offset": 6885982208, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.17.ffn_down.weight", "offset": 6988742656, "shape": [ 14336, 3584 ], "size": 102760448, "type": 1 }, { "name": "blk.17.attn_v.weight", "offset": 7091503104, "shape": [ 3584, 2048 ], "size": 102760448, "type": 1 }, { "name": "blk.17.attn_q.weight", "offset": 7106183168, "shape": [ 3584, 4096 ], "size": 14680064, "type": 1 }, { "name": "blk.18.attn_v.weight", "offset": 7135543296, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 7150223360, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 7150237696, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 7150252032, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.18.ffn_gate.weight", "offset": 7253012480, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.18.ffn_up.weight", "offset": 7355772928, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.18.post_attention_norm.weight", "offset": 7458533376, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 7458547712, "shape": [ 3584, 4096 ], "size": 14336, "type": 1 }, { "name": "blk.18.attn_output.weight", "offset": 7487907840, "shape": [ 4096, 3584 ], "size": 29360128, "type": 1 }, { "name": "blk.18.attn_k.weight", "offset": 7517267968, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.18.ffn_norm.weight", "offset": 7531948032, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 7531962368, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 7531976704, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 7531991040, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.19.ffn_gate.weight", "offset": 7634751488, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.19.ffn_up.weight", "offset": 7737511936, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.19.post_attention_norm.weight", "offset": 7840272384, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 7840286720, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 7840301056, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.19.attn_output.weight", "offset": 7854981120, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.19.attn_q.weight", "offset": 7884341248, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.19.attn_v.weight", "offset": 7913701376, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.20.attn_k.weight", "offset": 7928381440, "shape": [ 3584, 2048 ], "size": 14680064, "type": 1 }, { "name": "blk.20.attn_output.weight", "offset": 7943061504, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.20.attn_q.weight", "offset": 7972421632, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.20.attn_v.weight", "offset": 8001781760, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.20.attn_norm.weight", "offset": 8016461824, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 8016476160, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.20.ffn_gate.weight", "offset": 8119236608, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.20.ffn_up.weight", "offset": 8221997056, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.20.post_attention_norm.weight", "offset": 8324757504, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 8324771840, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 8324786176, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 8324800512, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 8427560960, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.21.ffn_gate.weight", "offset": 8427575296, "shape": [ 3584, 14336 ], "size": 14336, "type": 1 }, { "name": "blk.21.ffn_up.weight", "offset": 8530335744, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.21.post_attention_norm.weight", "offset": 8633096192, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 8633110528, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 8633124864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 8633139200, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.21.attn_output.weight", "offset": 8647819264, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.21.attn_q.weight", "offset": 8677179392, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.21.attn_v.weight", "offset": 8706539520, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 8721219584, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 8721233920, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.22.ffn_gate.weight", "offset": 8823994368, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.22.ffn_up.weight", "offset": 8926754816, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.22.post_attention_norm.weight", "offset": 9029515264, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 9029529600, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 9029543936, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 9029558272, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.22.attn_output.weight", "offset": 9044238336, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.22.attn_q.weight", "offset": 9073598464, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.22.attn_v.weight", "offset": 9102958592, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 9117638656, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 9117652992, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.23.ffn_gate.weight", "offset": 9220413440, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.23.ffn_up.weight", "offset": 9323173888, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.23.post_attention_norm.weight", "offset": 9425934336, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 9425948672, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 9425963008, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 9425977344, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.23.attn_output.weight", "offset": 9440657408, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.23.attn_q.weight", "offset": 9470017536, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.23.attn_v.weight", "offset": 9499377664, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 9514057728, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 9514072064, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.24.ffn_gate.weight", "offset": 9616832512, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.24.ffn_up.weight", "offset": 9719592960, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.24.post_attention_norm.weight", "offset": 9822353408, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 9822367744, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 9822382080, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 9822396416, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.24.attn_output.weight", "offset": 9837076480, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.24.attn_q.weight", "offset": 9866436608, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.24.attn_v.weight", "offset": 9895796736, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.25.attn_norm.weight", "offset": 9910476800, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 9910491136, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.25.ffn_gate.weight", "offset": 10013251584, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.25.ffn_up.weight", "offset": 10116012032, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.25.post_attention_norm.weight", "offset": 10218772480, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 10218786816, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 10218801152, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 10218815488, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.25.attn_output.weight", "offset": 10233495552, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.25.attn_q.weight", "offset": 10262855680, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.25.attn_v.weight", "offset": 10292215808, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 10306895872, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 10306910208, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.26.ffn_gate.weight", "offset": 10409670656, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.26.ffn_up.weight", "offset": 10512431104, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.26.post_attention_norm.weight", "offset": 10615191552, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 10615205888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 10615220224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 10615234560, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.26.attn_output.weight", "offset": 10629914624, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.26.attn_q.weight", "offset": 10659274752, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.26.attn_v.weight", "offset": 10688634880, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.27.attn_norm.weight", "offset": 10703314944, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 10703329280, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.27.ffn_gate.weight", "offset": 10806089728, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.27.ffn_up.weight", "offset": 10908850176, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.27.post_attention_norm.weight", "offset": 11011610624, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 11011624960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 11011639296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 11011653632, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.27.attn_output.weight", "offset": 11026333696, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.27.attn_q.weight", "offset": 11055693824, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.27.attn_v.weight", "offset": 11085053952, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.28.attn_norm.weight", "offset": 11099734016, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 11099748352, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.28.ffn_gate.weight", "offset": 11202508800, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.28.ffn_up.weight", "offset": 11305269248, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.28.post_attention_norm.weight", "offset": 11408029696, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 11408044032, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 11408058368, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 11408072704, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.28.attn_output.weight", "offset": 11422752768, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.28.attn_q.weight", "offset": 11452112896, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.28.attn_v.weight", "offset": 11481473024, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.29.attn_norm.weight", "offset": 11496153088, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 11496167424, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.29.ffn_gate.weight", "offset": 11598927872, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.29.ffn_up.weight", "offset": 11701688320, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.29.post_attention_norm.weight", "offset": 11804448768, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 11804463104, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 11804477440, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 11804491776, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.29.attn_output.weight", "offset": 11819171840, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.29.attn_q.weight", "offset": 11848531968, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.29.attn_v.weight", "offset": 11877892096, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.30.attn_norm.weight", "offset": 11892572160, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 11892586496, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.30.ffn_gate.weight", "offset": 11995346944, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.30.ffn_up.weight", "offset": 12098107392, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.30.post_attention_norm.weight", "offset": 12200867840, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 12200882176, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 12200896512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 12200910848, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.30.attn_output.weight", "offset": 12215590912, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.30.attn_q.weight", "offset": 12244951040, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.30.attn_v.weight", "offset": 12274311168, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.31.attn_k.weight", "offset": 12288991232, "shape": [ 3584, 2048 ], "size": 14680064, "type": 1 }, { "name": "blk.31.ffn_down.weight", "offset": 12303671296, "shape": [ 14336, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.31.ffn_gate.weight", "offset": 12406431744, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.31.ffn_up.weight", "offset": 12509192192, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.31.post_attention_norm.weight", "offset": 12611952640, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 12611966976, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 12611981312, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 12611995648, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 12612009984, "shape": [ 4096, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.31.attn_q.weight", "offset": 12641370112, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.31.attn_v.weight", "offset": 12670730240, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.32.ffn_gate.weight", "offset": 12685410304, "shape": [ 3584, 14336 ], "size": 14680064, "type": 1 }, { "name": "blk.32.ffn_up.weight", "offset": 12788170752, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.32.attn_k.weight", "offset": 12890931200, "shape": [ 3584, 2048 ], "size": 102760448, "type": 1 }, { "name": "blk.32.attn_output.weight", "offset": 12905611264, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.32.attn_q.weight", "offset": 12934971392, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.32.attn_v.weight", "offset": 12964331520, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.32.attn_norm.weight", "offset": 12979011584, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 12979025920, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.32.post_attention_norm.weight", "offset": 13081786368, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 13081800704, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 13081815040, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 13081829376, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 13081843712, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.33.ffn_gate.weight", "offset": 13184604160, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.33.ffn_up.weight", "offset": 13287364608, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.33.post_attention_norm.weight", "offset": 13390125056, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 13390139392, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 13390153728, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 13390168064, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.33.attn_output.weight", "offset": 13404848128, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.33.attn_q.weight", "offset": 13434208256, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.33.attn_v.weight", "offset": 13463568384, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.34.attn_norm.weight", "offset": 13478248448, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 13478262784, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.34.ffn_gate.weight", "offset": 13581023232, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.34.ffn_up.weight", "offset": 13683783680, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.34.post_attention_norm.weight", "offset": 13786544128, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 13786558464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 13786572800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 13786587136, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.34.attn_output.weight", "offset": 13801267200, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.34.attn_q.weight", "offset": 13830627328, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.34.attn_v.weight", "offset": 13859987456, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.35.attn_norm.weight", "offset": 13874667520, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 13874681856, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.35.ffn_gate.weight", "offset": 13977442304, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.35.ffn_up.weight", "offset": 14080202752, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.35.post_attention_norm.weight", "offset": 14182963200, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 14182977536, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 14182991872, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 14183006208, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.35.attn_output.weight", "offset": 14197686272, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.35.attn_q.weight", "offset": 14227046400, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.35.attn_v.weight", "offset": 14256406528, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.36.ffn_norm.weight", "offset": 14271086592, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 14271100928, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.36.ffn_gate.weight", "offset": 14373861376, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.36.ffn_up.weight", "offset": 14476621824, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.36.post_attention_norm.weight", "offset": 14579382272, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 14579396608, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 14579410944, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 14579425280, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.36.attn_output.weight", "offset": 14594105344, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.36.attn_q.weight", "offset": 14623465472, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.36.attn_v.weight", "offset": 14652825600, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.37.attn_norm.weight", "offset": 14667505664, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 14667520000, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.37.ffn_gate.weight", "offset": 14770280448, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.37.ffn_up.weight", "offset": 14873040896, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.37.post_attention_norm.weight", "offset": 14975801344, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 14975815680, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 14975830016, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 14975844352, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.37.attn_output.weight", "offset": 14990524416, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.37.attn_q.weight", "offset": 15019884544, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.37.attn_v.weight", "offset": 15049244672, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.38.attn_norm.weight", "offset": 15063924736, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 15063939072, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.38.ffn_gate.weight", "offset": 15166699520, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.38.ffn_up.weight", "offset": 15269459968, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.38.post_attention_norm.weight", "offset": 15372220416, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 15372234752, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 15372249088, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 15372263424, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.38.attn_output.weight", "offset": 15386943488, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.38.attn_q.weight", "offset": 15416303616, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.38.attn_v.weight", "offset": 15445663744, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.39.attn_norm.weight", "offset": 15460343808, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 15460358144, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.39.ffn_gate.weight", "offset": 15563118592, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.39.ffn_up.weight", "offset": 15665879040, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.39.post_attention_norm.weight", "offset": 15768639488, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 15768653824, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 15768668160, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 15768682496, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.39.attn_output.weight", "offset": 15783362560, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.39.attn_q.weight", "offset": 15812722688, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.39.attn_v.weight", "offset": 15842082816, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.40.attn_norm.weight", "offset": 15856762880, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 15856777216, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.40.ffn_gate.weight", "offset": 15959537664, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.40.ffn_up.weight", "offset": 16062298112, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.40.post_attention_norm.weight", "offset": 16165058560, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 16165072896, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 16165087232, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 16165101568, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.40.attn_output.weight", "offset": 16179781632, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.40.attn_q.weight", "offset": 16209141760, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.40.attn_v.weight", "offset": 16238501888, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "blk.41.attn_norm.weight", "offset": 16253181952, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 16253196288, "shape": [ 14336, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.41.ffn_gate.weight", "offset": 16355956736, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.41.ffn_up.weight", "offset": 16458717184, "shape": [ 3584, 14336 ], "size": 102760448, "type": 1 }, { "name": "blk.41.post_attention_norm.weight", "offset": 16561477632, "shape": [ 3584 ], "size": 102760448, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 16561491968, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 16561506304, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 16561520640, "shape": [ 3584, 2048 ], "size": 14336, "type": 1 }, { "name": "blk.41.attn_output.weight", "offset": 16576200704, "shape": [ 4096, 3584 ], "size": 14680064, "type": 1 }, { "name": "blk.41.attn_q.weight", "offset": 16605560832, "shape": [ 3584, 4096 ], "size": 29360128, "type": 1 }, { "name": "blk.41.attn_v.weight", "offset": 16634920960, "shape": [ 3584, 2048 ], "size": 29360128, "type": 1 }, { "name": "output_norm.weight", "offset": 16649601024, "shape": [ 3584 ], "size": 14680064, "type": 0 }, { "name": "token_embd.weight", "offset": 16649615360, "shape": [ 3584, 256000 ], "size": 14336, "type": 1 } ], "version": 3 }