moondream

{ "metadata": { "general.architecture": "phi2", "general.file_type": 7, "general.name": "moondream2", "general.quantization_version": 2, "phi2.attention.head_count": 32, "phi2.attention.head_count_kv": 32, "phi2.attention.layer_norm_epsilon": 0.00001, "phi2.block_count": 24, "phi2.context_length": 2048, "phi2.embedding_length": 2048, "phi2.feed_forward_length": 8192, "phi2.rope.dimension_count": 32, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.bos_token_id": 50256, "tokenizer.ggml.eos_token_id": 50256, "tokenizer.ggml.merges": "... (50000 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.token_type": "... (51200 values)", "tokenizer.ggml.tokens": "... (51200 values)", "tokenizer.ggml.unknown_token_id": 50256 }, "num_params": 1418270720, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 111411200, "shape": [ 2048 ], "size": 111411200, "type": 0 }, { "name": "blk.0.attn_norm.bias", "offset": 111419392, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.0.attn_qkv.weight", "offset": 111427584, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.0.attn_qkv.bias", "offset": 124796928, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 124821504, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.0.attn_output.bias", "offset": 129277952, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.0.ffn_up.weight", "offset": 129286144, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.0.ffn_up.bias", "offset": 147111936, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 147144704, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.0.ffn_down.bias", "offset": 164970496, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 164978688, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_norm.bias", "offset": 164986880, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_qkv.weight", "offset": 164995072, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.1.attn_qkv.bias", "offset": 178364416, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 178388992, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.1.attn_output.bias", "offset": 182845440, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.1.ffn_up.weight", "offset": 182853632, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.1.ffn_up.bias", "offset": 200679424, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 200712192, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.1.ffn_down.bias", "offset": 218537984, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.2.attn_norm.weight", "offset": 218546176, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_norm.bias", "offset": 218554368, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_qkv.weight", "offset": 218562560, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.2.attn_qkv.bias", "offset": 231931904, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 231956480, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.2.attn_output.bias", "offset": 236412928, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.2.ffn_up.weight", "offset": 236421120, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.2.ffn_up.bias", "offset": 254246912, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 254279680, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.2.ffn_down.bias", "offset": 272105472, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.3.attn_norm.weight", "offset": 272113664, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_norm.bias", "offset": 272121856, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_qkv.weight", "offset": 272130048, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.3.attn_qkv.bias", "offset": 285499392, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 285523968, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.3.attn_output.bias", "offset": 289980416, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.3.ffn_up.weight", "offset": 289988608, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.3.ffn_up.bias", "offset": 307814400, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 307847168, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.3.ffn_down.bias", "offset": 325672960, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 325681152, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_norm.bias", "offset": 325689344, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_qkv.weight", "offset": 325697536, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.4.attn_qkv.bias", "offset": 339066880, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 339091456, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.4.attn_output.bias", "offset": 343547904, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.4.ffn_up.weight", "offset": 343556096, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.4.ffn_up.bias", "offset": 361381888, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 361414656, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.4.ffn_down.bias", "offset": 379240448, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 379248640, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_norm.bias", "offset": 379256832, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_qkv.weight", "offset": 379265024, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.5.attn_qkv.bias", "offset": 392634368, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 392658944, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.5.attn_output.bias", "offset": 397115392, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.5.ffn_up.weight", "offset": 397123584, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.5.ffn_up.bias", "offset": 414949376, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 414982144, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.5.ffn_down.bias", "offset": 432807936, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 432816128, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_norm.bias", "offset": 432824320, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_qkv.weight", "offset": 432832512, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.6.attn_qkv.bias", "offset": 446201856, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.6.attn_output.weight", "offset": 446226432, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.6.attn_output.bias", "offset": 450682880, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.6.ffn_up.weight", "offset": 450691072, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.6.ffn_up.bias", "offset": 468516864, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 468549632, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.6.ffn_down.bias", "offset": 486375424, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 486383616, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.7.attn_norm.bias", "offset": 486391808, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.7.attn_qkv.weight", "offset": 486400000, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.7.attn_qkv.bias", "offset": 499769344, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 499793920, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.7.attn_output.bias", "offset": 504250368, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 504258560, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.7.ffn_up.bias", "offset": 522084352, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 522117120, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.7.ffn_down.bias", "offset": 539942912, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 539951104, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_norm.bias", "offset": 539959296, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_qkv.weight", "offset": 539967488, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.8.attn_qkv.bias", "offset": 553336832, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 553361408, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.8.attn_output.bias", "offset": 557817856, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.8.ffn_up.weight", "offset": 557826048, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.8.ffn_up.bias", "offset": 575651840, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 575684608, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.8.ffn_down.bias", "offset": 593510400, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.9.attn_norm.weight", "offset": 593518592, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_norm.bias", "offset": 593526784, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_qkv.weight", "offset": 593534976, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.9.attn_qkv.bias", "offset": 606904320, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 606928896, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.9.attn_output.bias", "offset": 611385344, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.9.ffn_up.weight", "offset": 611393536, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.9.ffn_up.bias", "offset": 629219328, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 629252096, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.9.ffn_down.bias", "offset": 647077888, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 647086080, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_norm.bias", "offset": 647094272, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_qkv.weight", "offset": 647102464, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.10.attn_qkv.bias", "offset": 660471808, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 660496384, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.10.attn_output.bias", "offset": 664952832, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 664961024, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.10.ffn_up.bias", "offset": 682786816, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 682819584, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.10.ffn_down.bias", "offset": 700645376, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.11.attn_norm.weight", "offset": 700653568, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_norm.bias", "offset": 700661760, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_qkv.weight", "offset": 700669952, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.11.attn_qkv.bias", "offset": 714039296, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 714063872, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.11.attn_output.bias", "offset": 718520320, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.11.ffn_up.weight", "offset": 718528512, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.11.ffn_up.bias", "offset": 736354304, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 736387072, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.11.ffn_down.bias", "offset": 754212864, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 754221056, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_norm.bias", "offset": 754229248, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_qkv.weight", "offset": 754237440, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.12.attn_qkv.bias", "offset": 767606784, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 767631360, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.12.attn_output.bias", "offset": 772087808, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 772096000, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.12.ffn_up.bias", "offset": 789921792, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 789954560, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.12.ffn_down.bias", "offset": 807780352, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 807788544, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_norm.bias", "offset": 807796736, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_qkv.weight", "offset": 807804928, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.13.attn_qkv.bias", "offset": 821174272, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 821198848, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.13.attn_output.bias", "offset": 825655296, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 825663488, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.13.ffn_up.bias", "offset": 843489280, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 843522048, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.13.ffn_down.bias", "offset": 861347840, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 861356032, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.14.attn_norm.bias", "offset": 861364224, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.14.attn_qkv.weight", "offset": 861372416, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.14.attn_qkv.bias", "offset": 874741760, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 874766336, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.14.attn_output.bias", "offset": 879222784, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.14.ffn_up.weight", "offset": 879230976, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.14.ffn_up.bias", "offset": 897056768, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 897089536, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.14.ffn_down.bias", "offset": 914915328, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 914923520, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_norm.bias", "offset": 914931712, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_qkv.weight", "offset": 914939904, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.15.attn_qkv.bias", "offset": 928309248, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 928333824, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.15.attn_output.bias", "offset": 932790272, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.15.ffn_up.weight", "offset": 932798464, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.15.ffn_up.bias", "offset": 950624256, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 950657024, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.15.ffn_down.bias", "offset": 968482816, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 968491008, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_norm.bias", "offset": 968499200, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_qkv.weight", "offset": 968507392, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.16.attn_qkv.bias", "offset": 981876736, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 981901312, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.16.attn_output.bias", "offset": 986357760, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 986365952, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.16.ffn_up.bias", "offset": 1004191744, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 1004224512, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.16.ffn_down.bias", "offset": 1022050304, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 1022058496, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_norm.bias", "offset": 1022066688, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_qkv.weight", "offset": 1022074880, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.17.attn_qkv.bias", "offset": 1035444224, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 1035468800, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.17.attn_output.bias", "offset": 1039925248, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 1039933440, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.17.ffn_up.bias", "offset": 1057759232, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 1057792000, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.17.ffn_down.bias", "offset": 1075617792, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 1075625984, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_norm.bias", "offset": 1075634176, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_qkv.weight", "offset": 1075642368, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.18.attn_qkv.bias", "offset": 1089011712, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 1089036288, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.18.attn_output.bias", "offset": 1093492736, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.18.ffn_up.weight", "offset": 1093500928, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.18.ffn_up.bias", "offset": 1111326720, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 1111359488, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.18.ffn_down.bias", "offset": 1129185280, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 1129193472, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_norm.bias", "offset": 1129201664, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_qkv.weight", "offset": 1129209856, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.19.attn_qkv.bias", "offset": 1142579200, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 1142603776, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.19.attn_output.bias", "offset": 1147060224, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.19.ffn_up.weight", "offset": 1147068416, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.19.ffn_up.bias", "offset": 1164894208, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 1164926976, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.19.ffn_down.bias", "offset": 1182752768, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.20.attn_norm.weight", "offset": 1182760960, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_norm.bias", "offset": 1182769152, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_qkv.weight", "offset": 1182777344, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.20.attn_qkv.bias", "offset": 1196146688, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 1196171264, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.20.attn_output.bias", "offset": 1200627712, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.20.ffn_up.weight", "offset": 1200635904, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.20.ffn_up.bias", "offset": 1218461696, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 1218494464, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.20.ffn_down.bias", "offset": 1236320256, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.21.attn_norm.weight", "offset": 1236328448, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_norm.bias", "offset": 1236336640, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_qkv.weight", "offset": 1236344832, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.21.attn_qkv.bias", "offset": 1249714176, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.21.attn_output.weight", "offset": 1249738752, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.21.attn_output.bias", "offset": 1254195200, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.21.ffn_up.weight", "offset": 1254203392, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.21.ffn_up.bias", "offset": 1272029184, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 1272061952, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.21.ffn_down.bias", "offset": 1289887744, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 1289895936, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.22.attn_norm.bias", "offset": 1289904128, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.22.attn_qkv.weight", "offset": 1289912320, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.22.attn_qkv.bias", "offset": 1303281664, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 1303306240, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.22.attn_output.bias", "offset": 1307762688, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.22.ffn_up.weight", "offset": 1307770880, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.22.ffn_up.bias", "offset": 1325596672, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 1325629440, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.22.ffn_down.bias", "offset": 1343455232, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 1343463424, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_norm.bias", "offset": 1343471616, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_qkv.weight", "offset": 1343479808, "shape": [ 2048, 6144 ], "size": 8192, "type": 8 }, { "name": "blk.23.attn_qkv.bias", "offset": 1356849152, "shape": [ 6144 ], "size": 13369344, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 1356873728, "shape": [ 2048, 2048 ], "size": 24576, "type": 8 }, { "name": "blk.23.attn_output.bias", "offset": 1361330176, "shape": [ 2048 ], "size": 4456448, "type": 0 }, { "name": "blk.23.ffn_up.weight", "offset": 1361338368, "shape": [ 2048, 8192 ], "size": 8192, "type": 8 }, { "name": "blk.23.ffn_up.bias", "offset": 1379164160, "shape": [ 8192 ], "size": 17825792, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 1379196928, "shape": [ 8192, 2048 ], "size": 32768, "type": 8 }, { "name": "blk.23.ffn_down.bias", "offset": 1397022720, "shape": [ 2048 ], "size": 17825792, "type": 0 }, { "name": "output_norm.weight", "offset": 1397030912, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "output_norm.bias", "offset": 1397039104, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "output.weight", "offset": 1397047296, "shape": [ 2048, 51200 ], "size": 8192, "type": 8 }, { "name": "output.bias", "offset": 1508458496, "shape": [ 51200 ], "size": 111411200, "type": 0 } ], "version": 3 }