moondream

{ "metadata": { "general.architecture": "phi2", "general.file_type": 2, "general.name": "moondream2", "general.quantization_version": 2, "phi2.attention.head_count": 32, "phi2.attention.head_count_kv": 32, "phi2.attention.layer_norm_epsilon": 0.00001, "phi2.block_count": 24, "phi2.context_length": 2048, "phi2.embedding_length": 2048, "phi2.feed_forward_length": 8192, "phi2.rope.dimension_count": 32, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.bos_token_id": 50256, "tokenizer.ggml.eos_token_id": 50256, "tokenizer.ggml.merges": "... (50000 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.token_type": "... (51200 values)", "tokenizer.ggml.tokens": "... (51200 values)", "tokenizer.ggml.unknown_token_id": 50256 }, "num_params": 1418270720, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 58982400, "shape": [ 2048 ], "size": 58982400, "type": 0 }, { "name": "blk.0.attn_norm.bias", "offset": 58990592, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.0.attn_qkv.weight", "offset": 58998784, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.0.attn_qkv.bias", "offset": 66076672, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 66101248, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.0.attn_output.bias", "offset": 68460544, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.0.ffn_up.weight", "offset": 68468736, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.0.ffn_up.bias", "offset": 77905920, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 77938688, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.0.ffn_down.bias", "offset": 87375872, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 87384064, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_norm.bias", "offset": 87392256, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_qkv.weight", "offset": 87400448, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.1.attn_qkv.bias", "offset": 94478336, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 94502912, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.1.attn_output.bias", "offset": 96862208, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.1.ffn_up.weight", "offset": 96870400, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.1.ffn_up.bias", "offset": 106307584, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 106340352, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.1.ffn_down.bias", "offset": 115777536, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.2.attn_norm.weight", "offset": 115785728, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_norm.bias", "offset": 115793920, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_qkv.weight", "offset": 115802112, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.2.attn_qkv.bias", "offset": 122880000, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 122904576, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.2.attn_output.bias", "offset": 125263872, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.2.ffn_up.weight", "offset": 125272064, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.2.ffn_up.bias", "offset": 134709248, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 134742016, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.2.ffn_down.bias", "offset": 144179200, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.3.attn_norm.weight", "offset": 144187392, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_norm.bias", "offset": 144195584, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_qkv.weight", "offset": 144203776, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.3.attn_qkv.bias", "offset": 151281664, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 151306240, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.3.attn_output.bias", "offset": 153665536, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.3.ffn_up.weight", "offset": 153673728, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.3.ffn_up.bias", "offset": 163110912, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 163143680, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.3.ffn_down.bias", "offset": 172580864, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 172589056, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_norm.bias", "offset": 172597248, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_qkv.weight", "offset": 172605440, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.4.attn_qkv.bias", "offset": 179683328, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 179707904, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.4.attn_output.bias", "offset": 182067200, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.4.ffn_up.weight", "offset": 182075392, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.4.ffn_up.bias", "offset": 191512576, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 191545344, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.4.ffn_down.bias", "offset": 200982528, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 200990720, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_norm.bias", "offset": 200998912, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_qkv.weight", "offset": 201007104, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.5.attn_qkv.bias", "offset": 208084992, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 208109568, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.5.attn_output.bias", "offset": 210468864, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.5.ffn_up.weight", "offset": 210477056, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.5.ffn_up.bias", "offset": 219914240, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 219947008, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.5.ffn_down.bias", "offset": 229384192, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 229392384, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_norm.bias", "offset": 229400576, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_qkv.weight", "offset": 229408768, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.6.attn_qkv.bias", "offset": 236486656, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.6.attn_output.weight", "offset": 236511232, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.6.attn_output.bias", "offset": 238870528, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.6.ffn_up.weight", "offset": 238878720, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.6.ffn_up.bias", "offset": 248315904, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 248348672, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.6.ffn_down.bias", "offset": 257785856, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 257794048, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.7.attn_norm.bias", "offset": 257802240, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.7.attn_qkv.weight", "offset": 257810432, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.7.attn_qkv.bias", "offset": 264888320, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 264912896, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.7.attn_output.bias", "offset": 267272192, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 267280384, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.7.ffn_up.bias", "offset": 276717568, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 276750336, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.7.ffn_down.bias", "offset": 286187520, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 286195712, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_norm.bias", "offset": 286203904, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_qkv.weight", "offset": 286212096, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.8.attn_qkv.bias", "offset": 293289984, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 293314560, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.8.attn_output.bias", "offset": 295673856, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.8.ffn_up.weight", "offset": 295682048, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.8.ffn_up.bias", "offset": 305119232, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 305152000, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.8.ffn_down.bias", "offset": 314589184, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.9.attn_norm.weight", "offset": 314597376, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_norm.bias", "offset": 314605568, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_qkv.weight", "offset": 314613760, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.9.attn_qkv.bias", "offset": 321691648, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 321716224, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.9.attn_output.bias", "offset": 324075520, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.9.ffn_up.weight", "offset": 324083712, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.9.ffn_up.bias", "offset": 333520896, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 333553664, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.9.ffn_down.bias", "offset": 342990848, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 342999040, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_norm.bias", "offset": 343007232, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_qkv.weight", "offset": 343015424, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.10.attn_qkv.bias", "offset": 350093312, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 350117888, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.10.attn_output.bias", "offset": 352477184, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 352485376, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.10.ffn_up.bias", "offset": 361922560, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 361955328, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.10.ffn_down.bias", "offset": 371392512, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.11.attn_norm.weight", "offset": 371400704, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_norm.bias", "offset": 371408896, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_qkv.weight", "offset": 371417088, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.11.attn_qkv.bias", "offset": 378494976, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 378519552, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.11.attn_output.bias", "offset": 380878848, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.11.ffn_up.weight", "offset": 380887040, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.11.ffn_up.bias", "offset": 390324224, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 390356992, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.11.ffn_down.bias", "offset": 399794176, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 399802368, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_norm.bias", "offset": 399810560, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_qkv.weight", "offset": 399818752, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.12.attn_qkv.bias", "offset": 406896640, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 406921216, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.12.attn_output.bias", "offset": 409280512, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 409288704, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.12.ffn_up.bias", "offset": 418725888, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 418758656, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.12.ffn_down.bias", "offset": 428195840, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 428204032, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_norm.bias", "offset": 428212224, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_qkv.weight", "offset": 428220416, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.13.attn_qkv.bias", "offset": 435298304, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 435322880, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.13.attn_output.bias", "offset": 437682176, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 437690368, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.13.ffn_up.bias", "offset": 447127552, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 447160320, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.13.ffn_down.bias", "offset": 456597504, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 456605696, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.14.attn_norm.bias", "offset": 456613888, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.14.attn_qkv.weight", "offset": 456622080, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.14.attn_qkv.bias", "offset": 463699968, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 463724544, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.14.attn_output.bias", "offset": 466083840, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.14.ffn_up.weight", "offset": 466092032, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.14.ffn_up.bias", "offset": 475529216, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 475561984, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.14.ffn_down.bias", "offset": 484999168, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 485007360, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_norm.bias", "offset": 485015552, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_qkv.weight", "offset": 485023744, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.15.attn_qkv.bias", "offset": 492101632, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 492126208, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.15.attn_output.bias", "offset": 494485504, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.15.ffn_up.weight", "offset": 494493696, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.15.ffn_up.bias", "offset": 503930880, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 503963648, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.15.ffn_down.bias", "offset": 513400832, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 513409024, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_norm.bias", "offset": 513417216, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_qkv.weight", "offset": 513425408, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.16.attn_qkv.bias", "offset": 520503296, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 520527872, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.16.attn_output.bias", "offset": 522887168, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 522895360, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.16.ffn_up.bias", "offset": 532332544, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 532365312, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.16.ffn_down.bias", "offset": 541802496, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 541810688, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_norm.bias", "offset": 541818880, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_qkv.weight", "offset": 541827072, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.17.attn_qkv.bias", "offset": 548904960, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 548929536, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.17.attn_output.bias", "offset": 551288832, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 551297024, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.17.ffn_up.bias", "offset": 560734208, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 560766976, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.17.ffn_down.bias", "offset": 570204160, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 570212352, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_norm.bias", "offset": 570220544, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_qkv.weight", "offset": 570228736, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.18.attn_qkv.bias", "offset": 577306624, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 577331200, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.18.attn_output.bias", "offset": 579690496, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.18.ffn_up.weight", "offset": 579698688, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.18.ffn_up.bias", "offset": 589135872, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 589168640, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.18.ffn_down.bias", "offset": 598605824, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 598614016, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_norm.bias", "offset": 598622208, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_qkv.weight", "offset": 598630400, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.19.attn_qkv.bias", "offset": 605708288, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 605732864, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.19.attn_output.bias", "offset": 608092160, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.19.ffn_up.weight", "offset": 608100352, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.19.ffn_up.bias", "offset": 617537536, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 617570304, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.19.ffn_down.bias", "offset": 627007488, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.20.attn_norm.weight", "offset": 627015680, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_norm.bias", "offset": 627023872, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_qkv.weight", "offset": 627032064, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.20.attn_qkv.bias", "offset": 634109952, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 634134528, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.20.attn_output.bias", "offset": 636493824, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.20.ffn_up.weight", "offset": 636502016, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.20.ffn_up.bias", "offset": 645939200, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 645971968, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.20.ffn_down.bias", "offset": 655409152, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.21.attn_norm.weight", "offset": 655417344, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_norm.bias", "offset": 655425536, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_qkv.weight", "offset": 655433728, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.21.attn_qkv.bias", "offset": 662511616, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.21.attn_output.weight", "offset": 662536192, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.21.attn_output.bias", "offset": 664895488, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.21.ffn_up.weight", "offset": 664903680, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.21.ffn_up.bias", "offset": 674340864, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 674373632, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.21.ffn_down.bias", "offset": 683810816, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 683819008, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.22.attn_norm.bias", "offset": 683827200, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.22.attn_qkv.weight", "offset": 683835392, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.22.attn_qkv.bias", "offset": 690913280, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 690937856, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.22.attn_output.bias", "offset": 693297152, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.22.ffn_up.weight", "offset": 693305344, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.22.ffn_up.bias", "offset": 702742528, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 702775296, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.22.ffn_down.bias", "offset": 712212480, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 712220672, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_norm.bias", "offset": 712228864, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_qkv.weight", "offset": 712237056, "shape": [ 2048, 6144 ], "size": 8192, "type": 2 }, { "name": "blk.23.attn_qkv.bias", "offset": 719314944, "shape": [ 6144 ], "size": 7077888, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 719339520, "shape": [ 2048, 2048 ], "size": 24576, "type": 2 }, { "name": "blk.23.attn_output.bias", "offset": 721698816, "shape": [ 2048 ], "size": 2359296, "type": 0 }, { "name": "blk.23.ffn_up.weight", "offset": 721707008, "shape": [ 2048, 8192 ], "size": 8192, "type": 2 }, { "name": "blk.23.ffn_up.bias", "offset": 731144192, "shape": [ 8192 ], "size": 9437184, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 731176960, "shape": [ 8192, 2048 ], "size": 32768, "type": 2 }, { "name": "blk.23.ffn_down.bias", "offset": 740614144, "shape": [ 2048 ], "size": 9437184, "type": 0 }, { "name": "output_norm.weight", "offset": 740622336, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "output_norm.bias", "offset": 740630528, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "output.weight", "offset": 740638720, "shape": [ 2048, 51200 ], "size": 8192, "type": 14 }, { "name": "output.bias", "offset": 826654720, "shape": [ 51200 ], "size": 86016000, "type": 0 } ], "version": 3 }