moondream

{ "metadata": { "clip.has_llava_projector": true, "clip.has_text_encoder": false, "clip.has_vision_encoder": true, "clip.projector_type": "mlp", "clip.use_gelu": true, "clip.vision.attention.head_count": 16, "clip.vision.attention.layer_norm_epsilon": 0.000001, "clip.vision.block_count": 28, "clip.vision.embedding_length": 1152, "clip.vision.feed_forward_length": 4304, "clip.vision.image_mean": [ 0.5, 0.5, 0.5 ], "clip.vision.image_size": 378, "clip.vision.image_std": [ 0.5, 0.5, 0.5 ], "clip.vision.patch_size": 14, "clip.vision.projection_dim": 2048, "general.architecture": "clip", "general.description": "image encoder for vikhyatk/moondream2", "general.file_type": 1, "general.name": "vikhyatk/moondream2" }, "num_params": 454451392, "tensors": [ { "name": "mm.0.bias", "offset": 18874368, "shape": [ 8192 ], "size": 18874368, "type": 0 }, { "name": "mm.2.weight", "offset": 18907136, "shape": [ 8192, 2048 ], "size": 32768, "type": 1 }, { "name": "mm.2.bias", "offset": 52461568, "shape": [ 2048 ], "size": 33554432, "type": 0 }, { "name": "v.position_embd.weight", "offset": 52469760, "shape": [ 1152, 729, 1 ], "size": 8192, "type": 1 }, { "name": "v.patch_embd.weight", "offset": 54149376, "shape": [ 14, 14, 3, 1152 ], "size": 1679616, "type": 1 }, { "name": "v.patch_embd.bias", "offset": 55504128, "shape": [ 1152 ], "size": 1354752, "type": 0 }, { "name": "v.post_ln.weight", "offset": 55508736, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.post_ln.bias", "offset": 55513344, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.0.attn_q.weight", "offset": 55517952, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.0.attn_q.bias", "offset": 58172160, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.0.attn_k.weight", "offset": 58176768, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.0.attn_k.bias", "offset": 60830976, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.0.attn_v.weight", "offset": 60835584, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.0.attn_v.bias", "offset": 63489792, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.0.attn_out.weight", "offset": 63494400, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.0.attn_out.bias", "offset": 66148608, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.0.ln1.weight", "offset": 66153216, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.0.ln1.bias", "offset": 66157824, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.0.ffn_down.weight", "offset": 66162432, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.0.ffn_down.bias", "offset": 76078848, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.0.ffn_up.weight", "offset": 76096064, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.0.ffn_up.bias", "offset": 86012480, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.0.ln2.weight", "offset": 86017088, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.0.ln2.bias", "offset": 86021696, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.1.attn_q.weight", "offset": 86026304, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.1.attn_q.bias", "offset": 88680512, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.1.attn_k.weight", "offset": 88685120, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.1.attn_k.bias", "offset": 91339328, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.1.attn_v.weight", "offset": 91343936, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.1.attn_v.bias", "offset": 93998144, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.1.attn_out.weight", "offset": 94002752, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.1.attn_out.bias", "offset": 96656960, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.1.ln1.weight", "offset": 96661568, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.1.ln1.bias", "offset": 96666176, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.1.ffn_down.weight", "offset": 96670784, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.1.ffn_down.bias", "offset": 106587200, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.1.ffn_up.weight", "offset": 106604416, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.1.ffn_up.bias", "offset": 116520832, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.1.ln2.weight", "offset": 116525440, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.1.ln2.bias", "offset": 116530048, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.2.attn_q.weight", "offset": 116534656, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.2.attn_q.bias", "offset": 119188864, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.2.attn_k.weight", "offset": 119193472, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.2.attn_k.bias", "offset": 121847680, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.2.attn_v.weight", "offset": 121852288, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.2.attn_v.bias", "offset": 124506496, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.2.attn_out.weight", "offset": 124511104, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.2.attn_out.bias", "offset": 127165312, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.2.ln1.weight", "offset": 127169920, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.2.ln1.bias", "offset": 127174528, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.2.ffn_down.weight", "offset": 127179136, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.2.ffn_down.bias", "offset": 137095552, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.2.ffn_up.weight", "offset": 137112768, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.2.ffn_up.bias", "offset": 147029184, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.2.ln2.weight", "offset": 147033792, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.2.ln2.bias", "offset": 147038400, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.3.attn_q.weight", "offset": 147043008, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.3.attn_q.bias", "offset": 149697216, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.3.attn_k.weight", "offset": 149701824, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.3.attn_k.bias", "offset": 152356032, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.3.attn_v.weight", "offset": 152360640, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.3.attn_v.bias", "offset": 155014848, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.3.attn_out.weight", "offset": 155019456, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.3.attn_out.bias", "offset": 157673664, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.3.ln1.weight", "offset": 157678272, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.3.ln1.bias", "offset": 157682880, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.3.ffn_down.weight", "offset": 157687488, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.3.ffn_down.bias", "offset": 167603904, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.3.ffn_up.weight", "offset": 167621120, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.3.ffn_up.bias", "offset": 177537536, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.3.ln2.weight", "offset": 177542144, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.3.ln2.bias", "offset": 177546752, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.4.attn_q.weight", "offset": 177551360, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.4.attn_q.bias", "offset": 180205568, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.4.attn_k.weight", "offset": 180210176, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.4.attn_k.bias", "offset": 182864384, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.4.attn_v.weight", "offset": 182868992, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.4.attn_v.bias", "offset": 185523200, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.4.attn_out.weight", "offset": 185527808, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.4.attn_out.bias", "offset": 188182016, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.4.ln1.weight", "offset": 188186624, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.4.ln1.bias", "offset": 188191232, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.4.ffn_down.weight", "offset": 188195840, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.4.ffn_down.bias", "offset": 198112256, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.4.ffn_up.weight", "offset": 198129472, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.4.ffn_up.bias", "offset": 208045888, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.4.ln2.weight", "offset": 208050496, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.4.ln2.bias", "offset": 208055104, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.5.attn_q.weight", "offset": 208059712, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.5.attn_q.bias", "offset": 210713920, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.5.attn_k.weight", "offset": 210718528, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.5.attn_k.bias", "offset": 213372736, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.5.attn_v.weight", "offset": 213377344, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.5.attn_v.bias", "offset": 216031552, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.5.attn_out.weight", "offset": 216036160, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.5.attn_out.bias", "offset": 218690368, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.5.ln1.weight", "offset": 218694976, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.5.ln1.bias", "offset": 218699584, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.5.ffn_down.weight", "offset": 218704192, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.5.ffn_down.bias", "offset": 228620608, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.5.ffn_up.weight", "offset": 228637824, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.5.ffn_up.bias", "offset": 238554240, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.5.ln2.weight", "offset": 238558848, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.5.ln2.bias", "offset": 238563456, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.6.attn_q.weight", "offset": 238568064, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.6.attn_q.bias", "offset": 241222272, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.6.attn_k.weight", "offset": 241226880, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.6.attn_k.bias", "offset": 243881088, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.6.attn_v.weight", "offset": 243885696, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.6.attn_v.bias", "offset": 246539904, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.6.attn_out.weight", "offset": 246544512, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.6.attn_out.bias", "offset": 249198720, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.6.ln1.weight", "offset": 249203328, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.6.ln1.bias", "offset": 249207936, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.6.ffn_down.weight", "offset": 249212544, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.6.ffn_down.bias", "offset": 259128960, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.6.ffn_up.weight", "offset": 259146176, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.6.ffn_up.bias", "offset": 269062592, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.6.ln2.weight", "offset": 269067200, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.6.ln2.bias", "offset": 269071808, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.7.attn_q.weight", "offset": 269076416, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.7.attn_q.bias", "offset": 271730624, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.7.attn_k.weight", "offset": 271735232, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.7.attn_k.bias", "offset": 274389440, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.7.attn_v.weight", "offset": 274394048, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.7.attn_v.bias", "offset": 277048256, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.7.attn_out.weight", "offset": 277052864, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.7.attn_out.bias", "offset": 279707072, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.7.ln1.weight", "offset": 279711680, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.7.ln1.bias", "offset": 279716288, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.7.ffn_down.weight", "offset": 279720896, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.7.ffn_down.bias", "offset": 289637312, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.7.ffn_up.weight", "offset": 289654528, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.7.ffn_up.bias", "offset": 299570944, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.7.ln2.weight", "offset": 299575552, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.7.ln2.bias", "offset": 299580160, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.8.attn_q.weight", "offset": 299584768, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.8.attn_q.bias", "offset": 302238976, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.8.attn_k.weight", "offset": 302243584, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.8.attn_k.bias", "offset": 304897792, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.8.attn_v.weight", "offset": 304902400, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.8.attn_v.bias", "offset": 307556608, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.8.attn_out.weight", "offset": 307561216, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.8.attn_out.bias", "offset": 310215424, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.8.ln1.weight", "offset": 310220032, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.8.ln1.bias", "offset": 310224640, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.8.ffn_down.weight", "offset": 310229248, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.8.ffn_down.bias", "offset": 320145664, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.8.ffn_up.weight", "offset": 320162880, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.8.ffn_up.bias", "offset": 330079296, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.8.ln2.weight", "offset": 330083904, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.8.ln2.bias", "offset": 330088512, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.9.attn_q.weight", "offset": 330093120, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.9.attn_q.bias", "offset": 332747328, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.9.attn_k.weight", "offset": 332751936, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.9.attn_k.bias", "offset": 335406144, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.9.attn_v.weight", "offset": 335410752, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.9.attn_v.bias", "offset": 338064960, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.9.attn_out.weight", "offset": 338069568, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.9.attn_out.bias", "offset": 340723776, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.9.ln1.weight", "offset": 340728384, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.9.ln1.bias", "offset": 340732992, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.9.ffn_down.weight", "offset": 340737600, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.9.ffn_down.bias", "offset": 350654016, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.9.ffn_up.weight", "offset": 350671232, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.9.ffn_up.bias", "offset": 360587648, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.9.ln2.weight", "offset": 360592256, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.9.ln2.bias", "offset": 360596864, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.10.attn_q.weight", "offset": 360601472, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.10.attn_q.bias", "offset": 363255680, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.10.attn_k.weight", "offset": 363260288, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.10.attn_k.bias", "offset": 365914496, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.10.attn_v.weight", "offset": 365919104, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.10.attn_v.bias", "offset": 368573312, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.10.attn_out.weight", "offset": 368577920, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.10.attn_out.bias", "offset": 371232128, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.10.ln1.weight", "offset": 371236736, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.10.ln1.bias", "offset": 371241344, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.10.ffn_down.weight", "offset": 371245952, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.10.ffn_down.bias", "offset": 381162368, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.10.ffn_up.weight", "offset": 381179584, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.10.ffn_up.bias", "offset": 391096000, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.10.ln2.weight", "offset": 391100608, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.10.ln2.bias", "offset": 391105216, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.11.attn_q.weight", "offset": 391109824, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.11.attn_q.bias", "offset": 393764032, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.11.attn_k.weight", "offset": 393768640, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.11.attn_k.bias", "offset": 396422848, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.11.attn_v.weight", "offset": 396427456, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.11.attn_v.bias", "offset": 399081664, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.11.attn_out.weight", "offset": 399086272, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.11.attn_out.bias", "offset": 401740480, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.11.ln1.weight", "offset": 401745088, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.11.ln1.bias", "offset": 401749696, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.11.ffn_down.weight", "offset": 401754304, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.11.ffn_down.bias", "offset": 411670720, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.11.ffn_up.weight", "offset": 411687936, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.11.ffn_up.bias", "offset": 421604352, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.11.ln2.weight", "offset": 421608960, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.11.ln2.bias", "offset": 421613568, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.12.attn_q.weight", "offset": 421618176, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.12.attn_q.bias", "offset": 424272384, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.12.attn_k.weight", "offset": 424276992, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.12.attn_k.bias", "offset": 426931200, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.12.attn_v.weight", "offset": 426935808, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.12.attn_v.bias", "offset": 429590016, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.12.attn_out.weight", "offset": 429594624, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.12.attn_out.bias", "offset": 432248832, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.12.ln1.weight", "offset": 432253440, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.12.ln1.bias", "offset": 432258048, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.12.ffn_down.weight", "offset": 432262656, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.12.ffn_down.bias", "offset": 442179072, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.12.ffn_up.weight", "offset": 442196288, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.12.ffn_up.bias", "offset": 452112704, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.12.ln2.weight", "offset": 452117312, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.12.ln2.bias", "offset": 452121920, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.13.attn_q.weight", "offset": 452126528, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.13.attn_q.bias", "offset": 454780736, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.13.attn_k.weight", "offset": 454785344, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.13.attn_k.bias", "offset": 457439552, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.13.attn_v.weight", "offset": 457444160, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.13.attn_v.bias", "offset": 460098368, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.13.attn_out.weight", "offset": 460102976, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.13.attn_out.bias", "offset": 462757184, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.13.ln1.weight", "offset": 462761792, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.13.ln1.bias", "offset": 462766400, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.13.ffn_down.weight", "offset": 462771008, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.13.ffn_down.bias", "offset": 472687424, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.13.ffn_up.weight", "offset": 472704640, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.13.ffn_up.bias", "offset": 482621056, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.13.ln2.weight", "offset": 482625664, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.13.ln2.bias", "offset": 482630272, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.14.attn_q.weight", "offset": 482634880, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.14.attn_q.bias", "offset": 485289088, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.14.attn_k.weight", "offset": 485293696, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.14.attn_k.bias", "offset": 487947904, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.14.attn_v.weight", "offset": 487952512, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.14.attn_v.bias", "offset": 490606720, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.14.attn_out.weight", "offset": 490611328, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.14.attn_out.bias", "offset": 493265536, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.14.ln1.weight", "offset": 493270144, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.14.ln1.bias", "offset": 493274752, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.14.ffn_down.weight", "offset": 493279360, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.14.ffn_down.bias", "offset": 503195776, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.14.ffn_up.weight", "offset": 503212992, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.14.ffn_up.bias", "offset": 513129408, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.14.ln2.weight", "offset": 513134016, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.14.ln2.bias", "offset": 513138624, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.15.attn_q.weight", "offset": 513143232, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.15.attn_q.bias", "offset": 515797440, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.15.attn_k.weight", "offset": 515802048, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.15.attn_k.bias", "offset": 518456256, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.15.attn_v.weight", "offset": 518460864, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.15.attn_v.bias", "offset": 521115072, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.15.attn_out.weight", "offset": 521119680, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.15.attn_out.bias", "offset": 523773888, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.15.ln1.weight", "offset": 523778496, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.15.ln1.bias", "offset": 523783104, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.15.ffn_down.weight", "offset": 523787712, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.15.ffn_down.bias", "offset": 533704128, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.15.ffn_up.weight", "offset": 533721344, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.15.ffn_up.bias", "offset": 543637760, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.15.ln2.weight", "offset": 543642368, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.15.ln2.bias", "offset": 543646976, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.16.attn_q.weight", "offset": 543651584, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.16.attn_q.bias", "offset": 546305792, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.16.attn_k.weight", "offset": 546310400, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.16.attn_k.bias", "offset": 548964608, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.16.attn_v.weight", "offset": 548969216, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.16.attn_v.bias", "offset": 551623424, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.16.attn_out.weight", "offset": 551628032, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.16.attn_out.bias", "offset": 554282240, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.16.ln1.weight", "offset": 554286848, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.16.ln1.bias", "offset": 554291456, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.16.ffn_down.weight", "offset": 554296064, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.16.ffn_down.bias", "offset": 564212480, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.16.ffn_up.weight", "offset": 564229696, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.16.ffn_up.bias", "offset": 574146112, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.16.ln2.weight", "offset": 574150720, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.16.ln2.bias", "offset": 574155328, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.17.attn_q.weight", "offset": 574159936, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.17.attn_q.bias", "offset": 576814144, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.17.attn_k.weight", "offset": 576818752, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.17.attn_k.bias", "offset": 579472960, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.17.attn_v.weight", "offset": 579477568, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.17.attn_v.bias", "offset": 582131776, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.17.attn_out.weight", "offset": 582136384, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.17.attn_out.bias", "offset": 584790592, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.17.ln1.weight", "offset": 584795200, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.17.ln1.bias", "offset": 584799808, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.17.ffn_down.weight", "offset": 584804416, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.17.ffn_down.bias", "offset": 594720832, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.17.ffn_up.weight", "offset": 594738048, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.17.ffn_up.bias", "offset": 604654464, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.17.ln2.weight", "offset": 604659072, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.17.ln2.bias", "offset": 604663680, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.18.attn_q.weight", "offset": 604668288, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.18.attn_q.bias", "offset": 607322496, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.18.attn_k.weight", "offset": 607327104, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.18.attn_k.bias", "offset": 609981312, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.18.attn_v.weight", "offset": 609985920, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.18.attn_v.bias", "offset": 612640128, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.18.attn_out.weight", "offset": 612644736, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.18.attn_out.bias", "offset": 615298944, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.18.ln1.weight", "offset": 615303552, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.18.ln1.bias", "offset": 615308160, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.18.ffn_down.weight", "offset": 615312768, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.18.ffn_down.bias", "offset": 625229184, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.18.ffn_up.weight", "offset": 625246400, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.18.ffn_up.bias", "offset": 635162816, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.18.ln2.weight", "offset": 635167424, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.18.ln2.bias", "offset": 635172032, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.19.attn_q.weight", "offset": 635176640, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.19.attn_q.bias", "offset": 637830848, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.19.attn_k.weight", "offset": 637835456, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.19.attn_k.bias", "offset": 640489664, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.19.attn_v.weight", "offset": 640494272, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.19.attn_v.bias", "offset": 643148480, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.19.attn_out.weight", "offset": 643153088, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.19.attn_out.bias", "offset": 645807296, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.19.ln1.weight", "offset": 645811904, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.19.ln1.bias", "offset": 645816512, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.19.ffn_down.weight", "offset": 645821120, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.19.ffn_down.bias", "offset": 655737536, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.19.ffn_up.weight", "offset": 655754752, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.19.ffn_up.bias", "offset": 665671168, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.19.ln2.weight", "offset": 665675776, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.19.ln2.bias", "offset": 665680384, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.20.attn_q.weight", "offset": 665684992, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.20.attn_q.bias", "offset": 668339200, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.20.attn_k.weight", "offset": 668343808, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.20.attn_k.bias", "offset": 670998016, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.20.attn_v.weight", "offset": 671002624, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.20.attn_v.bias", "offset": 673656832, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.20.attn_out.weight", "offset": 673661440, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.20.attn_out.bias", "offset": 676315648, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.20.ln1.weight", "offset": 676320256, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.20.ln1.bias", "offset": 676324864, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.20.ffn_down.weight", "offset": 676329472, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.20.ffn_down.bias", "offset": 686245888, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.20.ffn_up.weight", "offset": 686263104, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.20.ffn_up.bias", "offset": 696179520, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.20.ln2.weight", "offset": 696184128, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.20.ln2.bias", "offset": 696188736, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.21.attn_q.weight", "offset": 696193344, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.21.attn_q.bias", "offset": 698847552, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.21.attn_k.weight", "offset": 698852160, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.21.attn_k.bias", "offset": 701506368, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.21.attn_v.weight", "offset": 701510976, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.21.attn_v.bias", "offset": 704165184, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.21.attn_out.weight", "offset": 704169792, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.21.attn_out.bias", "offset": 706824000, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.21.ln1.weight", "offset": 706828608, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.21.ln1.bias", "offset": 706833216, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.21.ffn_down.weight", "offset": 706837824, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.21.ffn_down.bias", "offset": 716754240, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.21.ffn_up.weight", "offset": 716771456, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.21.ffn_up.bias", "offset": 726687872, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.21.ln2.weight", "offset": 726692480, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.21.ln2.bias", "offset": 726697088, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.22.attn_q.weight", "offset": 726701696, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.22.attn_q.bias", "offset": 729355904, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.22.attn_k.weight", "offset": 729360512, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.22.attn_k.bias", "offset": 732014720, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.22.attn_v.weight", "offset": 732019328, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.22.attn_v.bias", "offset": 734673536, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.22.attn_out.weight", "offset": 734678144, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.22.attn_out.bias", "offset": 737332352, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.22.ln1.weight", "offset": 737336960, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.22.ln1.bias", "offset": 737341568, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.22.ffn_down.weight", "offset": 737346176, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.22.ffn_down.bias", "offset": 747262592, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.22.ffn_up.weight", "offset": 747279808, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.22.ffn_up.bias", "offset": 757196224, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.22.ln2.weight", "offset": 757200832, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.22.ln2.bias", "offset": 757205440, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.23.attn_q.weight", "offset": 757210048, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.23.attn_q.bias", "offset": 759864256, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.23.attn_k.weight", "offset": 759868864, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.23.attn_k.bias", "offset": 762523072, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.23.attn_v.weight", "offset": 762527680, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.23.attn_v.bias", "offset": 765181888, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.23.attn_out.weight", "offset": 765186496, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.23.attn_out.bias", "offset": 767840704, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.23.ln1.weight", "offset": 767845312, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.23.ln1.bias", "offset": 767849920, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.23.ffn_down.weight", "offset": 767854528, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.23.ffn_down.bias", "offset": 777770944, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.23.ffn_up.weight", "offset": 777788160, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.23.ffn_up.bias", "offset": 787704576, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.23.ln2.weight", "offset": 787709184, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.23.ln2.bias", "offset": 787713792, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.24.attn_q.weight", "offset": 787718400, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.24.attn_q.bias", "offset": 790372608, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.24.attn_k.weight", "offset": 790377216, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.24.attn_k.bias", "offset": 793031424, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.24.attn_v.weight", "offset": 793036032, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.24.attn_v.bias", "offset": 795690240, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.24.attn_out.weight", "offset": 795694848, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.24.attn_out.bias", "offset": 798349056, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.24.ln1.weight", "offset": 798353664, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.24.ln1.bias", "offset": 798358272, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.24.ffn_down.weight", "offset": 798362880, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.24.ffn_down.bias", "offset": 808279296, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.24.ffn_up.weight", "offset": 808296512, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.24.ffn_up.bias", "offset": 818212928, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.24.ln2.weight", "offset": 818217536, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.24.ln2.bias", "offset": 818222144, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.25.attn_q.weight", "offset": 818226752, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.25.attn_q.bias", "offset": 820880960, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.25.attn_k.weight", "offset": 820885568, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.25.attn_k.bias", "offset": 823539776, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.25.attn_v.weight", "offset": 823544384, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.25.attn_v.bias", "offset": 826198592, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.25.attn_out.weight", "offset": 826203200, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.25.attn_out.bias", "offset": 828857408, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.25.ln1.weight", "offset": 828862016, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.25.ln1.bias", "offset": 828866624, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.25.ffn_down.weight", "offset": 828871232, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.25.ffn_down.bias", "offset": 838787648, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.25.ffn_up.weight", "offset": 838804864, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.25.ffn_up.bias", "offset": 848721280, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.25.ln2.weight", "offset": 848725888, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.25.ln2.bias", "offset": 848730496, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.26.attn_q.weight", "offset": 848735104, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.26.attn_q.bias", "offset": 851389312, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.26.attn_k.weight", "offset": 851393920, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.26.attn_k.bias", "offset": 854048128, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.26.attn_v.weight", "offset": 854052736, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.26.attn_v.bias", "offset": 856706944, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.26.attn_out.weight", "offset": 856711552, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.26.attn_out.bias", "offset": 859365760, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.26.ln1.weight", "offset": 859370368, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.26.ln1.bias", "offset": 859374976, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.26.ffn_down.weight", "offset": 859379584, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.26.ffn_down.bias", "offset": 869296000, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.26.ffn_up.weight", "offset": 869313216, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.26.ffn_up.bias", "offset": 879229632, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.26.ln2.weight", "offset": 879234240, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.26.ln2.bias", "offset": 879238848, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.27.attn_q.weight", "offset": 879243456, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.27.attn_q.bias", "offset": 881897664, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.27.attn_k.weight", "offset": 881902272, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.27.attn_k.bias", "offset": 884556480, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.27.attn_v.weight", "offset": 884561088, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.27.attn_v.bias", "offset": 887215296, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.27.attn_out.weight", "offset": 887219904, "shape": [ 1152, 1152 ], "size": 4608, "type": 1 }, { "name": "v.blk.27.attn_out.bias", "offset": 889874112, "shape": [ 1152 ], "size": 2654208, "type": 0 }, { "name": "v.blk.27.ln1.weight", "offset": 889878720, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.27.ln1.bias", "offset": 889883328, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.27.ffn_down.weight", "offset": 889887936, "shape": [ 1152, 4304 ], "size": 4608, "type": 1 }, { "name": "v.blk.27.ffn_down.bias", "offset": 899804352, "shape": [ 4304 ], "size": 9916416, "type": 0 }, { "name": "v.blk.27.ffn_up.weight", "offset": 899821568, "shape": [ 4304, 1152 ], "size": 17216, "type": 1 }, { "name": "v.blk.27.ffn_up.bias", "offset": 909737984, "shape": [ 1152 ], "size": 9916416, "type": 0 }, { "name": "v.blk.27.ln2.weight", "offset": 909742592, "shape": [ 1152 ], "size": 4608, "type": 0 }, { "name": "v.blk.27.ln2.bias", "offset": 909747200, "shape": [ 1152 ], "size": 4608, "type": 0 } ], "version": 3 }