orca-mini

{ "metadata": { "general.architecture": "llama", "general.file_type": 13, "general.name": "LLaMA v2", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 32, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 32, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32000 values)", "tokenizer.ggml.token_type": "... (32000 values)", "tokenizer.ggml.tokens": "... (32000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 6738415616, "tensors": [ { "name": "blk.0.attn_q.weight", "offset": 56320000, "shape": [ 4096, 4096 ], "size": 56320000, "type": 11 }, { "name": "blk.0.attn_k.weight", "offset": 63528960, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 70737920, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 82272256, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 93806592, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 113180672, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.0.ffn_down.weight", "offset": 132554752, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 163553280, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 163569664, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 163586048, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.1.attn_k.weight", "offset": 170795008, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 178003968, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 189538304, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 201072640, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 220446720, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.1.ffn_down.weight", "offset": 239820800, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 270819328, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 270835712, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 270852096, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.2.attn_k.weight", "offset": 278061056, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.2.attn_v.weight", "offset": 285270016, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 296804352, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 308338688, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 327712768, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.2.ffn_down.weight", "offset": 347086848, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 378085376, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 378101760, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 378118144, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.3.attn_k.weight", "offset": 385327104, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.3.attn_v.weight", "offset": 392536064, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 404070400, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 415604736, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 434978816, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.3.ffn_down.weight", "offset": 454352896, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 485351424, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 485367808, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 485384192, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.4.attn_k.weight", "offset": 492593152, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 499802112, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 511336448, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 522870784, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 542244864, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.4.ffn_down.weight", "offset": 561618944, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 592617472, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 592633856, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 592650240, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.5.attn_k.weight", "offset": 599859200, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 607068160, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 618602496, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 630136832, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 649510912, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.5.ffn_down.weight", "offset": 668884992, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 699883520, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 699899904, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 699916288, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.6.attn_k.weight", "offset": 707125248, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 714334208, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 725868544, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 737402880, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 756776960, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.6.ffn_down.weight", "offset": 776151040, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 807149568, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 807165952, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 807182336, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.7.attn_k.weight", "offset": 814391296, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 821600256, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 833134592, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 844668928, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 864043008, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.7.ffn_down.weight", "offset": 883417088, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 914415616, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 914432000, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 914448384, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.8.attn_k.weight", "offset": 921657344, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 928866304, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 940400640, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 951934976, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 971309056, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.8.ffn_down.weight", "offset": 990683136, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 1021681664, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 1021698048, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1021714432, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.9.attn_k.weight", "offset": 1028923392, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.9.attn_v.weight", "offset": 1036132352, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 1047666688, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 1059201024, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 1078575104, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.9.ffn_down.weight", "offset": 1097949184, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 1128947712, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 1128964096, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1128980480, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.10.attn_k.weight", "offset": 1136189440, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 1143398400, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 1154932736, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 1166467072, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 1185841152, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.10.ffn_down.weight", "offset": 1205215232, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 1236213760, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 1236230144, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1236246528, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.11.attn_k.weight", "offset": 1243455488, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.11.attn_v.weight", "offset": 1250664448, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 1262198784, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 1273733120, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1293107200, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.11.ffn_down.weight", "offset": 1312481280, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 1343479808, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1343496192, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1343512576, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.12.attn_k.weight", "offset": 1350721536, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.12.attn_v.weight", "offset": 1357930496, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 1369464832, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 1380999168, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 1400373248, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.12.ffn_down.weight", "offset": 1419747328, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 1450745856, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 1450762240, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1450778624, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.13.attn_k.weight", "offset": 1457987584, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.13.attn_v.weight", "offset": 1465196544, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 1476730880, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 1488265216, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 1507639296, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.13.ffn_down.weight", "offset": 1527013376, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 1558011904, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 1558028288, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 1558044672, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.14.attn_k.weight", "offset": 1565253632, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 1572462592, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 1583996928, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 1595531264, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 1614905344, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.14.ffn_down.weight", "offset": 1634279424, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 1665277952, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 1665294336, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 1665310720, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.15.attn_k.weight", "offset": 1672519680, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 1679728640, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 1691262976, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 1702797312, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 1722171392, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.15.ffn_down.weight", "offset": 1741545472, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 1772544000, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 1772560384, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1772576768, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.16.attn_k.weight", "offset": 1779785728, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 1786994688, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 1798529024, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 1810063360, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 1829437440, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.16.ffn_down.weight", "offset": 1848811520, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 1879810048, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 1879826432, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 1879842816, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.17.attn_k.weight", "offset": 1887051776, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.17.attn_v.weight", "offset": 1894260736, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 1905795072, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 1917329408, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 1936703488, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.17.ffn_down.weight", "offset": 1956077568, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 1987076096, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 1987092480, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 1987108864, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.18.attn_k.weight", "offset": 1994317824, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 2001526784, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 2013061120, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 2024595456, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 2043969536, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.18.ffn_down.weight", "offset": 2063343616, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 2094342144, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 2094358528, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 2094374912, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.19.attn_k.weight", "offset": 2101583872, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 2108792832, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 2120327168, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 2131861504, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 2151235584, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.19.ffn_down.weight", "offset": 2170609664, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 2201608192, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 2201624576, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 2201640960, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.20.attn_k.weight", "offset": 2208849920, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 2216058880, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 2227593216, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 2239127552, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 2258501632, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.20.ffn_down.weight", "offset": 2277875712, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 2308874240, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 2308890624, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 2308907008, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.21.attn_k.weight", "offset": 2316115968, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 2323324928, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 2334859264, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.21.ffn_gate.weight", "offset": 2346393600, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 2365767680, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.21.ffn_down.weight", "offset": 2385141760, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 2416140288, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 2416156672, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 2416173056, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.22.attn_k.weight", "offset": 2423382016, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 2430590976, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 2442125312, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 2453659648, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 2473033728, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.22.ffn_down.weight", "offset": 2492407808, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 2523406336, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 2523422720, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 2523439104, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.23.attn_k.weight", "offset": 2530648064, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 2537857024, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 2549391360, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 2560925696, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 2580299776, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.23.ffn_down.weight", "offset": 2599673856, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 2630672384, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 2630688768, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 2630705152, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.24.attn_k.weight", "offset": 2637914112, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 2645123072, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 2656657408, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.24.ffn_gate.weight", "offset": 2668191744, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 2687565824, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.24.ffn_down.weight", "offset": 2706939904, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 2737938432, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 2737954816, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 2737971200, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.25.attn_k.weight", "offset": 2745180160, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 2752389120, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 2763923456, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 2775457792, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 2794831872, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.25.ffn_down.weight", "offset": 2814205952, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 2845204480, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 2845220864, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 2845237248, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.26.attn_k.weight", "offset": 2852446208, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 2859655168, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 2871189504, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 2882723840, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 2902097920, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.26.ffn_down.weight", "offset": 2921472000, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 2952470528, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 2952486912, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 2952503296, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.27.attn_k.weight", "offset": 2959712256, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 2966921216, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.27.attn_output.weight", "offset": 2978455552, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 2989989888, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 3009363968, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.27.ffn_down.weight", "offset": 3028738048, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 3059736576, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 3059752960, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 3059769344, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.28.attn_k.weight", "offset": 3066978304, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 3074187264, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.28.attn_output.weight", "offset": 3085721600, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.28.ffn_gate.weight", "offset": 3097255936, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 3116630016, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.28.ffn_down.weight", "offset": 3136004096, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 3167002624, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 3167019008, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 3167035392, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.29.attn_k.weight", "offset": 3174244352, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 3181453312, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.29.attn_output.weight", "offset": 3192987648, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.29.ffn_gate.weight", "offset": 3204521984, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 3223896064, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.29.ffn_down.weight", "offset": 3243270144, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 3274268672, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 3274285056, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 3274301440, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.30.attn_k.weight", "offset": 3281510400, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 3288719360, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.30.attn_output.weight", "offset": 3300253696, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.30.ffn_gate.weight", "offset": 3311788032, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 3331162112, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.30.ffn_down.weight", "offset": 3350536192, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 3381534720, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 3381551104, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 3381567488, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.31.attn_k.weight", "offset": 3388776448, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 3395985408, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.31.attn_output.weight", "offset": 3407519744, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.31.ffn_gate.weight", "offset": 3419054080, "shape": [ 4096, 11008 ], "size": 11534336, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 3438428160, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.31.ffn_down.weight", "offset": 3457802240, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 3488800768, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 3488817152, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "output_norm.weight", "offset": 3488833536, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "output.weight", "offset": 3488849920, "shape": [ 4096, 32000 ], "size": 16384, "type": 14 } ], "version": 2 }