yi

{ "metadata": { "general.architecture": "llama", "general.file_type": 2, "general.name": "Yi-1.5-6B", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 4, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 32, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 5000000, "llama.vocab_size": 64000, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (64000 values)", "tokenizer.ggml.token_type": "... (64000 values)", "tokenizer.ggml.tokens": "... (64000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 6061035520, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 147456000, "shape": [ 4096 ], "size": 147456000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 147472384, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.0.ffn_gate.weight", "offset": 172834816, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 198197248, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 223559680, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 223576064, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.0.attn_output.weight", "offset": 224755712, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.0.attn_q.weight", "offset": 234192896, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.0.attn_v.weight", "offset": 243630080, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 244809728, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 244826112, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.1.ffn_gate.weight", "offset": 270188544, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 295550976, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 320913408, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 320929792, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.1.attn_output.weight", "offset": 322109440, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.1.attn_q.weight", "offset": 331546624, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.1.attn_v.weight", "offset": 340983808, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 342163456, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 342179840, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.10.ffn_gate.weight", "offset": 367542272, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 392904704, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 418267136, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 418283520, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.10.attn_output.weight", "offset": 419463168, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.10.attn_q.weight", "offset": 428900352, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.10.attn_v.weight", "offset": 438337536, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 439517184, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 439533568, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.11.ffn_gate.weight", "offset": 464896000, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 490258432, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 515620864, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 515637248, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.11.attn_output.weight", "offset": 516816896, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.11.attn_q.weight", "offset": 526254080, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.11.attn_v.weight", "offset": 535691264, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.12.ffn_gate.weight", "offset": 536870912, "shape": [ 4096, 11008 ], "size": 1179648, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 562233344, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.12.attn_k.weight", "offset": 587595776, "shape": [ 4096, 512 ], "size": 25362432, "type": 2 }, { "name": "blk.12.attn_output.weight", "offset": 588775424, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.12.attn_q.weight", "offset": 598212608, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.12.attn_v.weight", "offset": 607649792, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 608829440, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 608845824, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.2.ffn_gate.weight", "offset": 634208256, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 659570688, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 684933120, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 684949504, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.2.attn_output.weight", "offset": 686129152, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.2.attn_q.weight", "offset": 695566336, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.2.attn_v.weight", "offset": 705003520, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 706183168, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 706199552, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.3.ffn_gate.weight", "offset": 731561984, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 756924416, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 782286848, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 782303232, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.3.attn_output.weight", "offset": 783482880, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.3.attn_q.weight", "offset": 792920064, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.3.attn_v.weight", "offset": 802357248, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 803536896, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 803553280, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.4.ffn_gate.weight", "offset": 828915712, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 854278144, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 879640576, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 879656960, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.4.attn_output.weight", "offset": 880836608, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.4.attn_q.weight", "offset": 890273792, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.4.attn_v.weight", "offset": 899710976, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 900890624, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 900907008, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.5.ffn_gate.weight", "offset": 926269440, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 951631872, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 976994304, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 977010688, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.5.attn_output.weight", "offset": 978190336, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.5.attn_q.weight", "offset": 987627520, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.5.attn_v.weight", "offset": 997064704, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 998244352, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 998260736, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.6.ffn_gate.weight", "offset": 1023623168, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 1048985600, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 1074348032, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1074364416, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 1075544064, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.6.attn_q.weight", "offset": 1084981248, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.6.attn_v.weight", "offset": 1094418432, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.7.attn_norm.weight", "offset": 1095598080, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1095614464, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.7.ffn_gate.weight", "offset": 1120976896, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 1146339328, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 1171701760, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1171718144, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.7.attn_output.weight", "offset": 1172897792, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.7.attn_q.weight", "offset": 1182334976, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.7.attn_v.weight", "offset": 1191772160, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 1192951808, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1192968192, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.8.ffn_gate.weight", "offset": 1218330624, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 1243693056, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 1269055488, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1269071872, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.8.attn_output.weight", "offset": 1270251520, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.8.attn_q.weight", "offset": 1279688704, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.8.attn_v.weight", "offset": 1289125888, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 1290305536, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1290321920, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.9.ffn_gate.weight", "offset": 1315684352, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 1341046784, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 1366409216, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1366425600, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.9.attn_output.weight", "offset": 1367605248, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.9.attn_q.weight", "offset": 1377042432, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.9.attn_v.weight", "offset": 1386479616, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 1387659264, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1387675648, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 1413038080, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 1413054464, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1413070848, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.13.ffn_gate.weight", "offset": 1438433280, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 1463795712, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 1489158144, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1489174528, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.13.attn_output.weight", "offset": 1490354176, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.13.attn_q.weight", "offset": 1499791360, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.13.attn_v.weight", "offset": 1509228544, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 1510408192, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1510424576, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.14.ffn_gate.weight", "offset": 1535787008, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 1561149440, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 1586511872, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1586528256, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.14.attn_output.weight", "offset": 1587707904, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.14.attn_q.weight", "offset": 1597145088, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.14.attn_v.weight", "offset": 1606582272, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 1607761920, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1607778304, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.15.ffn_gate.weight", "offset": 1633140736, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 1658503168, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 1683865600, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1683881984, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.15.attn_output.weight", "offset": 1685061632, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.15.attn_q.weight", "offset": 1694498816, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.15.attn_v.weight", "offset": 1703936000, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 1705115648, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 1705132032, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.16.ffn_gate.weight", "offset": 1730494464, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 1755856896, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 1781219328, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1781235712, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.16.attn_output.weight", "offset": 1782415360, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.16.attn_q.weight", "offset": 1791852544, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.16.attn_v.weight", "offset": 1801289728, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 1802469376, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 1802485760, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.17.ffn_gate.weight", "offset": 1827848192, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 1853210624, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 1878573056, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 1878589440, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.17.attn_output.weight", "offset": 1879769088, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.17.attn_q.weight", "offset": 1889206272, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.17.attn_v.weight", "offset": 1898643456, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 1899823104, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 1899839488, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.18.ffn_gate.weight", "offset": 1925201920, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 1950564352, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 1975926784, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 1975943168, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.18.attn_output.weight", "offset": 1977122816, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.18.attn_q.weight", "offset": 1986560000, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.18.attn_v.weight", "offset": 1995997184, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.19.attn_norm.weight", "offset": 1997176832, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 1997193216, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.19.ffn_gate.weight", "offset": 2022555648, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 2047918080, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.19.ffn_norm.weight", "offset": 2073280512, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2073296896, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.19.attn_output.weight", "offset": 2074476544, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.19.attn_q.weight", "offset": 2083913728, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.19.attn_v.weight", "offset": 2093350912, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 2094530560, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2094546944, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.20.ffn_gate.weight", "offset": 2119909376, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 2145271808, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 2170634240, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 2170650624, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.20.attn_output.weight", "offset": 2171830272, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.20.attn_q.weight", "offset": 2181267456, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.20.attn_v.weight", "offset": 2190704640, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 2191884288, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2191900672, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.21.ffn_gate.weight", "offset": 2217263104, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 2242625536, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 2267987968, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2268004352, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 2269184000, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.21.attn_q.weight", "offset": 2278621184, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.21.attn_v.weight", "offset": 2288058368, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.22.attn_norm.weight", "offset": 2289238016, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2289254400, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.22.ffn_gate.weight", "offset": 2314616832, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.22.ffn_up.weight", "offset": 2339979264, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 2365341696, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2365358080, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.22.attn_output.weight", "offset": 2366537728, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.22.attn_q.weight", "offset": 2375974912, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.22.attn_v.weight", "offset": 2385412096, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.23.attn_norm.weight", "offset": 2386591744, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2386608128, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.23.ffn_gate.weight", "offset": 2411970560, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 2437332992, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 2462695424, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2462711808, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.23.attn_output.weight", "offset": 2463891456, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.23.attn_q.weight", "offset": 2473328640, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.23.attn_v.weight", "offset": 2482765824, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.24.attn_norm.weight", "offset": 2483945472, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2483961856, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.24.ffn_gate.weight", "offset": 2509324288, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.24.ffn_up.weight", "offset": 2534686720, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.24.ffn_norm.weight", "offset": 2560049152, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 2560065536, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.24.attn_output.weight", "offset": 2561245184, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.24.attn_q.weight", "offset": 2570682368, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.24.attn_v.weight", "offset": 2580119552, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.25.attn_norm.weight", "offset": 2581299200, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 2581315584, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.25.ffn_gate.weight", "offset": 2606678016, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.25.ffn_up.weight", "offset": 2632040448, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.25.ffn_norm.weight", "offset": 2657402880, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 2657419264, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.25.attn_output.weight", "offset": 2658598912, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.25.attn_q.weight", "offset": 2668036096, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.25.attn_v.weight", "offset": 2677473280, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 2678652928, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 2678669312, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.26.ffn_gate.weight", "offset": 2704031744, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.26.ffn_up.weight", "offset": 2729394176, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.26.ffn_norm.weight", "offset": 2754756608, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 2754772992, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.26.attn_output.weight", "offset": 2755952640, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.26.attn_q.weight", "offset": 2765389824, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.26.attn_v.weight", "offset": 2774827008, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.27.attn_k.weight", "offset": 2776006656, "shape": [ 4096, 512 ], "size": 1179648, "type": 2 }, { "name": "blk.27.attn_q.weight", "offset": 2777186304, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.27.attn_v.weight", "offset": 2786623488, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "output.weight", "offset": 2787803136, "shape": [ 4096, 64000 ], "size": 1179648, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 3002843136, "shape": [ 4096 ], "size": 215040000, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3002859520, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.27.ffn_gate.weight", "offset": 3028221952, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.27.ffn_up.weight", "offset": 3053584384, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.27.ffn_norm.weight", "offset": 3078946816, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 3078963200, "shape": [ 4096, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.28.attn_norm.weight", "offset": 3088400384, "shape": [ 4096 ], "size": 9437184, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3088416768, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.28.ffn_gate.weight", "offset": 3113779200, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.28.ffn_up.weight", "offset": 3139141632, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.28.ffn_norm.weight", "offset": 3164504064, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3164520448, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.28.attn_output.weight", "offset": 3165700096, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.28.attn_q.weight", "offset": 3175137280, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.28.attn_v.weight", "offset": 3184574464, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.29.attn_norm.weight", "offset": 3185754112, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3185770496, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.29.ffn_gate.weight", "offset": 3211132928, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.29.ffn_up.weight", "offset": 3236495360, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.29.ffn_norm.weight", "offset": 3261857792, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 3261874176, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.29.attn_output.weight", "offset": 3263053824, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.29.attn_q.weight", "offset": 3272491008, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.29.attn_v.weight", "offset": 3281928192, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.30.attn_norm.weight", "offset": 3283107840, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3283124224, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.30.ffn_gate.weight", "offset": 3308486656, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.30.ffn_up.weight", "offset": 3333849088, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.30.ffn_norm.weight", "offset": 3359211520, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 3359227904, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.30.attn_output.weight", "offset": 3360407552, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.30.attn_q.weight", "offset": 3369844736, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.30.attn_v.weight", "offset": 3379281920, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "blk.31.attn_norm.weight", "offset": 3380461568, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 3380477952, "shape": [ 11008, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.31.ffn_gate.weight", "offset": 3405840384, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.31.ffn_up.weight", "offset": 3431202816, "shape": [ 4096, 11008 ], "size": 25362432, "type": 2 }, { "name": "blk.31.ffn_norm.weight", "offset": 3456565248, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 3456581632, "shape": [ 4096, 512 ], "size": 16384, "type": 2 }, { "name": "blk.31.attn_output.weight", "offset": 3457761280, "shape": [ 4096, 4096 ], "size": 1179648, "type": 2 }, { "name": "blk.31.attn_q.weight", "offset": 3467198464, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.31.attn_v.weight", "offset": 3476635648, "shape": [ 4096, 512 ], "size": 9437184, "type": 2 }, { "name": "output_norm.weight", "offset": 3477815296, "shape": [ 4096 ], "size": 1179648, "type": 0 } ], "version": 3 }