yi

{ "metadata": { "general.architecture": "llama", "general.file_type": 8, "general.name": "Yi-1.5-6B-Chat", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 4, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 32, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 5000000, "llama.vocab_size": 64000, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (64000 values)", "tokenizer.ggml.token_type": "... (64000 values)", "tokenizer.ggml.tokens": "... (64000 values)" }, "num_params": 6061035520, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 180224000, "shape": [ 4096 ], "size": 180224000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 180240384, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.0.ffn_gate.weight", "offset": 211238912, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.0.ffn_up.weight", "offset": 242237440, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.0.ffn_norm.weight", "offset": 273235968, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 273252352, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.0.attn_output.weight", "offset": 274694144, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.0.attn_q.weight", "offset": 286228480, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.0.attn_v.weight", "offset": 297762816, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.1.attn_norm.weight", "offset": 299204608, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 299220992, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.1.ffn_gate.weight", "offset": 330219520, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.1.ffn_up.weight", "offset": 361218048, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.1.ffn_norm.weight", "offset": 392216576, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 392232960, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.1.attn_output.weight", "offset": 393674752, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.1.attn_q.weight", "offset": 405209088, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.1.attn_v.weight", "offset": 416743424, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.10.attn_norm.weight", "offset": 418185216, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 418201600, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.10.ffn_gate.weight", "offset": 449200128, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.10.ffn_up.weight", "offset": 480198656, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.10.ffn_norm.weight", "offset": 511197184, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 511213568, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.10.attn_output.weight", "offset": 512655360, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.10.attn_q.weight", "offset": 524189696, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.10.attn_v.weight", "offset": 535724032, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.11.attn_norm.weight", "offset": 537165824, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 537182208, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.11.ffn_gate.weight", "offset": 568180736, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.11.ffn_up.weight", "offset": 599179264, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.11.ffn_norm.weight", "offset": 630177792, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 630194176, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.11.attn_output.weight", "offset": 631635968, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.11.attn_q.weight", "offset": 643170304, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.11.attn_v.weight", "offset": 654704640, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.12.ffn_gate.weight", "offset": 656146432, "shape": [ 4096, 11008 ], "size": 1441792, "type": 6 }, { "name": "blk.12.ffn_up.weight", "offset": 687144960, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.12.attn_k.weight", "offset": 718143488, "shape": [ 4096, 512 ], "size": 30998528, "type": 6 }, { "name": "blk.12.attn_output.weight", "offset": 719585280, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.12.attn_q.weight", "offset": 731119616, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.12.attn_v.weight", "offset": 742653952, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.2.attn_norm.weight", "offset": 744095744, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 744112128, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.2.ffn_gate.weight", "offset": 775110656, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.2.ffn_up.weight", "offset": 806109184, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.2.ffn_norm.weight", "offset": 837107712, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 837124096, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.2.attn_output.weight", "offset": 838565888, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.2.attn_q.weight", "offset": 850100224, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.2.attn_v.weight", "offset": 861634560, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.3.attn_norm.weight", "offset": 863076352, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 863092736, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.3.ffn_gate.weight", "offset": 894091264, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.3.ffn_up.weight", "offset": 925089792, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.3.ffn_norm.weight", "offset": 956088320, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 956104704, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.3.attn_output.weight", "offset": 957546496, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.3.attn_q.weight", "offset": 969080832, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.3.attn_v.weight", "offset": 980615168, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.4.attn_norm.weight", "offset": 982056960, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 982073344, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.4.ffn_gate.weight", "offset": 1013071872, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.4.ffn_up.weight", "offset": 1044070400, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.4.ffn_norm.weight", "offset": 1075068928, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1075085312, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.4.attn_output.weight", "offset": 1076527104, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.4.attn_q.weight", "offset": 1088061440, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.4.attn_v.weight", "offset": 1099595776, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.5.attn_norm.weight", "offset": 1101037568, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1101053952, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.5.ffn_gate.weight", "offset": 1132052480, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.5.ffn_up.weight", "offset": 1163051008, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.5.ffn_norm.weight", "offset": 1194049536, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1194065920, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.5.attn_output.weight", "offset": 1195507712, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.5.attn_q.weight", "offset": 1207042048, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.5.attn_v.weight", "offset": 1218576384, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.6.attn_norm.weight", "offset": 1220018176, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1220034560, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.6.ffn_gate.weight", "offset": 1251033088, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.6.ffn_up.weight", "offset": 1282031616, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.6.ffn_norm.weight", "offset": 1313030144, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1313046528, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.6.attn_output.weight", "offset": 1314488320, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.6.attn_q.weight", "offset": 1326022656, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.6.attn_v.weight", "offset": 1337556992, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.7.attn_norm.weight", "offset": 1338998784, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1339015168, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.7.ffn_gate.weight", "offset": 1370013696, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.7.ffn_up.weight", "offset": 1401012224, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.7.ffn_norm.weight", "offset": 1432010752, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1432027136, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.7.attn_output.weight", "offset": 1433468928, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.7.attn_q.weight", "offset": 1445003264, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.7.attn_v.weight", "offset": 1456537600, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.8.attn_norm.weight", "offset": 1457979392, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1457995776, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.8.ffn_gate.weight", "offset": 1488994304, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.8.ffn_up.weight", "offset": 1519992832, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.8.ffn_norm.weight", "offset": 1550991360, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1551007744, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.8.attn_output.weight", "offset": 1552449536, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.8.attn_q.weight", "offset": 1563983872, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.8.attn_v.weight", "offset": 1575518208, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.9.attn_norm.weight", "offset": 1576960000, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1576976384, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.9.ffn_gate.weight", "offset": 1607974912, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.9.ffn_up.weight", "offset": 1638973440, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.9.ffn_norm.weight", "offset": 1669971968, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1669988352, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.9.attn_output.weight", "offset": 1671430144, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.9.attn_q.weight", "offset": 1682964480, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.9.attn_v.weight", "offset": 1694498816, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.12.attn_norm.weight", "offset": 1695940608, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1695956992, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.12.ffn_norm.weight", "offset": 1726955520, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 1726971904, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1726988288, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.13.ffn_gate.weight", "offset": 1757986816, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.13.ffn_up.weight", "offset": 1788985344, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.13.ffn_norm.weight", "offset": 1819983872, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1820000256, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.13.attn_output.weight", "offset": 1821442048, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.13.attn_q.weight", "offset": 1832976384, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.13.attn_v.weight", "offset": 1844510720, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.14.attn_norm.weight", "offset": 1845952512, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1845968896, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.14.ffn_gate.weight", "offset": 1876967424, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.14.ffn_up.weight", "offset": 1907965952, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.14.ffn_norm.weight", "offset": 1938964480, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1938980864, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.14.attn_output.weight", "offset": 1940422656, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.14.attn_q.weight", "offset": 1951956992, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.14.attn_v.weight", "offset": 1963491328, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.15.attn_norm.weight", "offset": 1964933120, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1964949504, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.15.ffn_gate.weight", "offset": 1995948032, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.15.ffn_up.weight", "offset": 2026946560, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.15.ffn_norm.weight", "offset": 2057945088, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2057961472, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.15.attn_output.weight", "offset": 2059403264, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.15.attn_q.weight", "offset": 2070937600, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.15.attn_v.weight", "offset": 2082471936, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.16.attn_norm.weight", "offset": 2083913728, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2083930112, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.16.ffn_gate.weight", "offset": 2114928640, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.16.ffn_up.weight", "offset": 2145927168, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.16.ffn_norm.weight", "offset": 2176925696, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2176942080, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.16.attn_output.weight", "offset": 2178383872, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.16.attn_q.weight", "offset": 2189918208, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.16.attn_v.weight", "offset": 2201452544, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.17.attn_norm.weight", "offset": 2202894336, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2202910720, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.17.ffn_gate.weight", "offset": 2233909248, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.17.ffn_up.weight", "offset": 2264907776, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.17.ffn_norm.weight", "offset": 2295906304, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2295922688, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.17.attn_output.weight", "offset": 2297364480, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.17.attn_q.weight", "offset": 2308898816, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.17.attn_v.weight", "offset": 2320433152, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.18.attn_norm.weight", "offset": 2321874944, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2321891328, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.18.ffn_gate.weight", "offset": 2352889856, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.18.ffn_up.weight", "offset": 2383888384, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.18.ffn_norm.weight", "offset": 2414886912, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2414903296, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.18.attn_output.weight", "offset": 2416345088, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.18.attn_q.weight", "offset": 2427879424, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.18.attn_v.weight", "offset": 2439413760, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.19.attn_norm.weight", "offset": 2440855552, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2440871936, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.19.ffn_gate.weight", "offset": 2471870464, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.19.ffn_up.weight", "offset": 2502868992, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.19.ffn_norm.weight", "offset": 2533867520, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2533883904, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.19.attn_output.weight", "offset": 2535325696, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.19.attn_q.weight", "offset": 2546860032, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.19.attn_v.weight", "offset": 2558394368, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.20.attn_norm.weight", "offset": 2559836160, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2559852544, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.20.ffn_gate.weight", "offset": 2590851072, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.20.ffn_up.weight", "offset": 2621849600, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.20.ffn_norm.weight", "offset": 2652848128, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 2652864512, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.20.attn_output.weight", "offset": 2654306304, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.20.attn_q.weight", "offset": 2665840640, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.20.attn_v.weight", "offset": 2677374976, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.21.attn_norm.weight", "offset": 2678816768, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2678833152, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.21.ffn_gate.weight", "offset": 2709831680, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.21.ffn_up.weight", "offset": 2740830208, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.21.ffn_norm.weight", "offset": 2771828736, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2771845120, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.21.attn_output.weight", "offset": 2773286912, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.21.attn_q.weight", "offset": 2784821248, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.21.attn_v.weight", "offset": 2796355584, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.22.attn_norm.weight", "offset": 2797797376, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2797813760, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.22.ffn_gate.weight", "offset": 2828812288, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.22.ffn_up.weight", "offset": 2859810816, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.22.ffn_norm.weight", "offset": 2890809344, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2890825728, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.22.attn_output.weight", "offset": 2892267520, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.22.attn_q.weight", "offset": 2903801856, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.22.attn_v.weight", "offset": 2915336192, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.23.attn_norm.weight", "offset": 2916777984, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2916794368, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.23.ffn_gate.weight", "offset": 2947792896, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.23.ffn_up.weight", "offset": 2978791424, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.23.ffn_norm.weight", "offset": 3009789952, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3009806336, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.23.attn_output.weight", "offset": 3011248128, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.23.attn_q.weight", "offset": 3022782464, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.23.attn_v.weight", "offset": 3034316800, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.24.attn_norm.weight", "offset": 3035758592, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3035774976, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.24.ffn_gate.weight", "offset": 3066773504, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.24.ffn_up.weight", "offset": 3097772032, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.24.ffn_norm.weight", "offset": 3128770560, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 3128786944, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.24.attn_output.weight", "offset": 3130228736, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.24.attn_q.weight", "offset": 3141763072, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.24.attn_v.weight", "offset": 3153297408, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.25.attn_norm.weight", "offset": 3154739200, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3154755584, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.25.ffn_gate.weight", "offset": 3185754112, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.25.ffn_up.weight", "offset": 3216752640, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.25.ffn_norm.weight", "offset": 3247751168, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3247767552, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.25.attn_output.weight", "offset": 3249209344, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.25.attn_q.weight", "offset": 3260743680, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.25.attn_v.weight", "offset": 3272278016, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.26.attn_norm.weight", "offset": 3273719808, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3273736192, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.26.ffn_gate.weight", "offset": 3304734720, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.26.ffn_up.weight", "offset": 3335733248, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.26.ffn_norm.weight", "offset": 3366731776, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3366748160, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.26.attn_output.weight", "offset": 3368189952, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.26.attn_q.weight", "offset": 3379724288, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.26.attn_v.weight", "offset": 3391258624, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.27.attn_k.weight", "offset": 3392700416, "shape": [ 4096, 512 ], "size": 1441792, "type": 6 }, { "name": "blk.27.attn_q.weight", "offset": 3394142208, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.27.attn_v.weight", "offset": 3405676544, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "output.weight", "offset": 3407118336, "shape": [ 4096, 64000 ], "size": 1441792, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 3622158336, "shape": [ 4096 ], "size": 215040000, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3622174720, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.27.ffn_gate.weight", "offset": 3653173248, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.27.ffn_up.weight", "offset": 3684171776, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.27.ffn_norm.weight", "offset": 3715170304, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 3715186688, "shape": [ 4096, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.28.attn_norm.weight", "offset": 3726721024, "shape": [ 4096 ], "size": 11534336, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3726737408, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.28.ffn_gate.weight", "offset": 3757735936, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.28.ffn_up.weight", "offset": 3788734464, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.28.ffn_norm.weight", "offset": 3819732992, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3819749376, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.28.attn_output.weight", "offset": 3821191168, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.28.attn_q.weight", "offset": 3832725504, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.28.attn_v.weight", "offset": 3844259840, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.29.attn_norm.weight", "offset": 3845701632, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3845718016, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.29.ffn_gate.weight", "offset": 3876716544, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.29.ffn_up.weight", "offset": 3907715072, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.29.ffn_norm.weight", "offset": 3938713600, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 3938729984, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.29.attn_output.weight", "offset": 3940171776, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.29.attn_q.weight", "offset": 3951706112, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.29.attn_v.weight", "offset": 3963240448, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.30.attn_norm.weight", "offset": 3964682240, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3964698624, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.30.ffn_gate.weight", "offset": 3995697152, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.30.ffn_up.weight", "offset": 4026695680, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.30.ffn_norm.weight", "offset": 4057694208, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 4057710592, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.30.attn_output.weight", "offset": 4059152384, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.30.attn_q.weight", "offset": 4070686720, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.30.attn_v.weight", "offset": 4082221056, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "blk.31.attn_norm.weight", "offset": 4083662848, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 4083679232, "shape": [ 11008, 4096 ], "size": 16384, "type": 6 }, { "name": "blk.31.ffn_gate.weight", "offset": 4114677760, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.31.ffn_up.weight", "offset": 4145676288, "shape": [ 4096, 11008 ], "size": 30998528, "type": 6 }, { "name": "blk.31.ffn_norm.weight", "offset": 4176674816, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 4176691200, "shape": [ 4096, 512 ], "size": 16384, "type": 6 }, { "name": "blk.31.attn_output.weight", "offset": 4178132992, "shape": [ 4096, 4096 ], "size": 1441792, "type": 6 }, { "name": "blk.31.attn_q.weight", "offset": 4189667328, "shape": [ 4096, 4096 ], "size": 11534336, "type": 6 }, { "name": "blk.31.attn_v.weight", "offset": 4201201664, "shape": [ 4096, 512 ], "size": 11534336, "type": 6 }, { "name": "output_norm.weight", "offset": 4202643456, "shape": [ 4096 ], "size": 1441792, "type": 0 } ], "version": 3 }