yi

{ "metadata": { "general.architecture": "llama", "general.file_type": 17, "general.name": "Yi-1.5-6B-Chat", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 4, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 32, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 5000000, "llama.vocab_size": 64000, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (64000 values)", "tokenizer.ggml.token_type": "... (64000 values)", "tokenizer.ggml.tokens": "... (64000 values)" }, "num_params": 6061035520, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 180224000, "shape": [ 4096 ], "size": 180224000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 180240384, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 217227264, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 248225792, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.0.ffn_norm.weight", "offset": 279224320, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 279240704, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 280682496, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 292216832, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.0.attn_v.weight", "offset": 303751168, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 305471488, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 305487872, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 342474752, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 373473280, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 404471808, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 404488192, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 405929984, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 417464320, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.1.attn_v.weight", "offset": 428998656, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 430718976, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 430735360, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 467722240, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.10.ffn_up.weight", "offset": 498720768, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 529719296, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 529735680, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 531177472, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 542711808, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.10.attn_v.weight", "offset": 554246144, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 555966464, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 555982848, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.11.ffn_gate.weight", "offset": 592969728, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.11.ffn_up.weight", "offset": 623968256, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 654966784, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 654983168, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 656424960, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 667959296, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.11.attn_v.weight", "offset": 679493632, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.12.ffn_gate.weight", "offset": 681213952, "shape": [ 4096, 11008 ], "size": 1720320, "type": 13 }, { "name": "blk.12.ffn_up.weight", "offset": 712212480, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.12.attn_k.weight", "offset": 743211008, "shape": [ 4096, 512 ], "size": 30998528, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 744652800, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 756187136, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.12.attn_v.weight", "offset": 767721472, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 769163264, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 769179648, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 800178176, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 831176704, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 862175232, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 862191616, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 863633408, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 875167744, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.2.attn_v.weight", "offset": 886702080, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 888143872, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 888160256, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 919158784, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 950157312, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 981155840, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 981172224, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 982614016, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 994148352, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.3.attn_v.weight", "offset": 1005682688, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1007403008, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1007419392, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 1044406272, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 1075404800, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 1106403328, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1106419712, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 1107861504, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 1119395840, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.4.attn_v.weight", "offset": 1130930176, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 1132371968, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1132388352, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 1163386880, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 1194385408, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 1225383936, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1225400320, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 1226842112, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 1238376448, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.5.attn_v.weight", "offset": 1249910784, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 1251352576, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1251368960, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 1282367488, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 1313366016, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 1344364544, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1344380928, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 1345822720, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 1357357056, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.6.attn_v.weight", "offset": 1368891392, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 1370611712, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1370628096, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 1407614976, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.7.ffn_up.weight", "offset": 1438613504, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 1469612032, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1469628416, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 1471070208, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 1482604544, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.7.attn_v.weight", "offset": 1494138880, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 1495580672, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1495597056, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 1526595584, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 1557594112, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 1588592640, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1588609024, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 1590050816, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 1601585152, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.8.attn_v.weight", "offset": 1613119488, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 1614561280, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1614577664, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 1645576192, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.9.ffn_up.weight", "offset": 1676574720, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.9.ffn_norm.weight", "offset": 1707573248, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1707589632, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 1709031424, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 1720565760, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.9.attn_v.weight", "offset": 1732100096, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 1733820416, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1733836800, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.12.ffn_norm.weight", "offset": 1770823680, "shape": [ 4096 ], "size": 36986880, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 1770840064, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1770856448, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 1801854976, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.13.ffn_up.weight", "offset": 1832853504, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 1863852032, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1863868416, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 1865310208, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 1876844544, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.13.attn_v.weight", "offset": 1888378880, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 1889820672, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1889837056, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 1920835584, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 1951834112, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 1982832640, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1982849024, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 1984290816, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 1995825152, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.14.attn_v.weight", "offset": 2007359488, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 2008801280, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2008817664, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.15.ffn_gate.weight", "offset": 2045804544, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 2076803072, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 2107801600, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2107817984, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 2109259776, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 2120794112, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.15.attn_v.weight", "offset": 2132328448, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 2134048768, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2134065152, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 2165063680, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.16.ffn_up.weight", "offset": 2196062208, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 2227060736, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2227077120, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 2228518912, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 2240053248, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.16.attn_v.weight", "offset": 2251587584, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 2253029376, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2253045760, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 2284044288, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.17.ffn_up.weight", "offset": 2315042816, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 2346041344, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2346057728, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 2347499520, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 2359033856, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.17.attn_v.weight", "offset": 2370568192, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 2372009984, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2372026368, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.18.ffn_gate.weight", "offset": 2409013248, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 2440011776, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 2471010304, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2471026688, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 2472468480, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 2484002816, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.18.attn_v.weight", "offset": 2495537152, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 2497257472, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2497273856, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 2528272384, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.19.ffn_up.weight", "offset": 2559270912, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 2590269440, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2590285824, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 2591727616, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 2603261952, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.19.attn_v.weight", "offset": 2614796288, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 2616238080, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2616254464, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 2647252992, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 2678251520, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 2709250048, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 2709266432, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 2710708224, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 2722242560, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.20.attn_v.weight", "offset": 2733776896, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 2735218688, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2735235072, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.21.ffn_gate.weight", "offset": 2772221952, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 2803220480, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 2834219008, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2834235392, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 2835677184, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 2847211520, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.21.attn_v.weight", "offset": 2858745856, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 2860466176, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2860482560, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 2891481088, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.22.ffn_up.weight", "offset": 2922479616, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 2953478144, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2953494528, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 2954936320, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 2966470656, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.22.attn_v.weight", "offset": 2978004992, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 2979446784, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2979463168, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 3010461696, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 3041460224, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 3072458752, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3072475136, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 3073916928, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.23.attn_q.weight", "offset": 3085451264, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.23.attn_v.weight", "offset": 3096985600, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 3098427392, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3098443776, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.24.ffn_gate.weight", "offset": 3135430656, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 3166429184, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 3197427712, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 3197444096, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 3198885888, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.24.attn_q.weight", "offset": 3210420224, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.24.attn_v.weight", "offset": 3221954560, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 3223674880, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3223691264, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 3254689792, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.25.ffn_up.weight", "offset": 3285688320, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 3316686848, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3316703232, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 3318145024, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 3329679360, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.25.attn_v.weight", "offset": 3341213696, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 3342655488, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3342671872, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 3373670400, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 3404668928, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 3435667456, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3435683840, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 3437125632, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 3448659968, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.26.attn_v.weight", "offset": 3460194304, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.27.attn_k.weight", "offset": 3461636096, "shape": [ 4096, 512 ], "size": 1441792, "type": 13 }, { "name": "blk.27.attn_q.weight", "offset": 3463077888, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.27.attn_v.weight", "offset": 3474612224, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "output.weight", "offset": 3476332544, "shape": [ 4096, 64000 ], "size": 1720320, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 3691372544, "shape": [ 4096 ], "size": 215040000, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3691388928, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 3728375808, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 3759374336, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.27.ffn_norm.weight", "offset": 3790372864, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 3790389248, "shape": [ 4096, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 3801923584, "shape": [ 4096 ], "size": 11534336, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3801939968, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.28.ffn_gate.weight", "offset": 3838926848, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.28.ffn_up.weight", "offset": 3869925376, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.28.ffn_norm.weight", "offset": 3900923904, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3900940288, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.28.attn_output.weight", "offset": 3902382080, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 3913916416, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.28.attn_v.weight", "offset": 3925450752, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.29.attn_norm.weight", "offset": 3927171072, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3927187456, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.29.ffn_gate.weight", "offset": 3964174336, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.29.ffn_up.weight", "offset": 3995172864, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.29.ffn_norm.weight", "offset": 4026171392, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 4026187776, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.29.attn_output.weight", "offset": 4027629568, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 4039163904, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.29.attn_v.weight", "offset": 4050698240, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.30.attn_norm.weight", "offset": 4052418560, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 4052434944, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.30.ffn_gate.weight", "offset": 4089421824, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.30.ffn_up.weight", "offset": 4120420352, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.30.ffn_norm.weight", "offset": 4151418880, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 4151435264, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.30.attn_output.weight", "offset": 4152877056, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 4164411392, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.30.attn_v.weight", "offset": 4175945728, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "blk.31.attn_norm.weight", "offset": 4177666048, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 4177682432, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.31.ffn_gate.weight", "offset": 4214669312, "shape": [ 4096, 11008 ], "size": 36986880, "type": 13 }, { "name": "blk.31.ffn_up.weight", "offset": 4245667840, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.31.ffn_norm.weight", "offset": 4276666368, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 4276682752, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.31.attn_output.weight", "offset": 4278124544, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.31.attn_q.weight", "offset": 4289658880, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.31.attn_v.weight", "offset": 4301193216, "shape": [ 4096, 512 ], "size": 11534336, "type": 14 }, { "name": "output_norm.weight", "offset": 4302913536, "shape": [ 4096 ], "size": 1720320, "type": 0 } ], "version": 3 }