yi

{ "metadata": { "general.architecture": "llama", "general.file_type": 16, "general.name": "Yi-1.5-9B-Chat", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 4, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 48, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 5000000, "llama.vocab_size": 64000, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (64000 values)", "tokenizer.ggml.token_type": "... (64000 values)", "tokenizer.ggml.tokens": "... (64000 values)" }, "num_params": 8829407232, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 180224000, "shape": [ 4096 ], "size": 180224000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 180240384, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 211238912, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 242237440, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.0.ffn_norm.weight", "offset": 273235968, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 273252352, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 274694144, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 286228480, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.0.attn_v.weight", "offset": 297762816, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 299204608, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 299220992, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 330219520, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 361218048, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 392216576, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 392232960, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 393674752, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 405209088, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.1.attn_v.weight", "offset": 416743424, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 418185216, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 418201600, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 449200128, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.10.ffn_up.weight", "offset": 480198656, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 511197184, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 511213568, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 512655360, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 524189696, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.10.attn_v.weight", "offset": 535724032, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 537165824, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 537182208, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 568180736, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.11.ffn_up.weight", "offset": 599179264, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 630177792, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 630194176, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 631635968, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 643170304, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.11.attn_v.weight", "offset": 654704640, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 656146432, "shape": [ 4096, 11008 ], "size": 1441792, "type": 13 }, { "name": "blk.12.ffn_up.weight", "offset": 687144960, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.12.attn_k.weight", "offset": 718143488, "shape": [ 4096, 512 ], "size": 30998528, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 719585280, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 731119616, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.12.attn_v.weight", "offset": 742653952, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 744095744, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 744112128, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 775110656, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 806109184, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 837107712, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 837124096, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 838565888, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 850100224, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.2.attn_v.weight", "offset": 861634560, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 863076352, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 863092736, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 894091264, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 925089792, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 956088320, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 956104704, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 957546496, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 969080832, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.3.attn_v.weight", "offset": 980615168, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 982056960, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 982073344, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 1013071872, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 1044070400, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 1075068928, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1075085312, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 1076527104, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 1088061440, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.4.attn_v.weight", "offset": 1099595776, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 1101037568, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1101053952, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 1132052480, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 1163051008, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 1194049536, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1194065920, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 1195507712, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 1207042048, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.5.attn_v.weight", "offset": 1218576384, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 1220018176, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1220034560, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 1251033088, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 1282031616, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 1313030144, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1313046528, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 1314488320, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 1326022656, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.6.attn_v.weight", "offset": 1337556992, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 1338998784, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1339015168, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 1370013696, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.7.ffn_up.weight", "offset": 1401012224, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 1432010752, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1432027136, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 1433468928, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 1445003264, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.7.attn_v.weight", "offset": 1456537600, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 1457979392, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1457995776, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 1488994304, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 1519992832, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 1550991360, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1551007744, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 1552449536, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 1563983872, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.8.attn_v.weight", "offset": 1575518208, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 1576960000, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1576976384, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 1607974912, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.9.ffn_up.weight", "offset": 1638973440, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.9.ffn_norm.weight", "offset": 1669971968, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1669988352, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 1671430144, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 1682964480, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.9.attn_v.weight", "offset": 1694498816, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 1695940608, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1695956992, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 1726955520, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 1726971904, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1726988288, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 1757986816, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.13.ffn_up.weight", "offset": 1788985344, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 1819983872, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1820000256, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 1821442048, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 1832976384, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.13.attn_v.weight", "offset": 1844510720, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 1845952512, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1845968896, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 1876967424, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 1907965952, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 1938964480, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1938980864, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 1940422656, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 1951956992, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.14.attn_v.weight", "offset": 1963491328, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 1964933120, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1964949504, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 1995948032, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 2026946560, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 2057945088, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2057961472, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 2059403264, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 2070937600, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.15.attn_v.weight", "offset": 2082471936, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 2083913728, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2083930112, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 2114928640, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.16.ffn_up.weight", "offset": 2145927168, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 2176925696, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2176942080, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 2178383872, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 2189918208, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.16.attn_v.weight", "offset": 2201452544, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 2202894336, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2202910720, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 2233909248, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.17.ffn_up.weight", "offset": 2264907776, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 2295906304, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2295922688, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 2297364480, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 2308898816, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.17.attn_v.weight", "offset": 2320433152, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 2321874944, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2321891328, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 2352889856, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 2383888384, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 2414886912, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2414903296, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 2416345088, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 2427879424, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.18.attn_v.weight", "offset": 2439413760, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 2440855552, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2440871936, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 2471870464, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.19.ffn_up.weight", "offset": 2502868992, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 2533867520, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2533883904, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 2535325696, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 2546860032, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.19.attn_v.weight", "offset": 2558394368, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 2559836160, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2559852544, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 2590851072, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 2621849600, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 2652848128, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 2652864512, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 2654306304, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 2665840640, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.20.attn_v.weight", "offset": 2677374976, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 2678816768, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2678833152, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.21.ffn_gate.weight", "offset": 2709831680, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 2740830208, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 2771828736, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2771845120, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 2773286912, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 2784821248, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.21.attn_v.weight", "offset": 2796355584, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 2797797376, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2797813760, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 2828812288, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.22.ffn_up.weight", "offset": 2859810816, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 2890809344, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2890825728, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 2892267520, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 2903801856, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.22.attn_v.weight", "offset": 2915336192, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 2916777984, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2916794368, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 2947792896, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 2978791424, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 3009789952, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3009806336, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 3011248128, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.23.attn_q.weight", "offset": 3022782464, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.23.attn_v.weight", "offset": 3034316800, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 3035758592, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3035774976, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.24.ffn_gate.weight", "offset": 3066773504, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 3097772032, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 3128770560, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 3128786944, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 3130228736, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.24.attn_q.weight", "offset": 3141763072, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.24.attn_v.weight", "offset": 3153297408, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 3154739200, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3154755584, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 3185754112, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.25.ffn_up.weight", "offset": 3216752640, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 3247751168, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3247767552, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 3249209344, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 3260743680, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.25.attn_v.weight", "offset": 3272278016, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 3273719808, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3273736192, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 3304734720, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 3335733248, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 3366731776, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3366748160, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 3368189952, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 3379724288, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.26.attn_v.weight", "offset": 3391258624, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.27.attn_k.weight", "offset": 3392700416, "shape": [ 4096, 512 ], "size": 1441792, "type": 13 }, { "name": "blk.27.attn_q.weight", "offset": 3394142208, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.27.attn_v.weight", "offset": 3405676544, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 3407118336, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3407134720, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 3438133248, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 3469131776, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.27.ffn_norm.weight", "offset": 3500130304, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 3500146688, "shape": [ 4096, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 3511681024, "shape": [ 4096 ], "size": 11534336, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3511697408, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.28.ffn_gate.weight", "offset": 3542695936, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.28.ffn_up.weight", "offset": 3573694464, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.28.ffn_norm.weight", "offset": 3604692992, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3604709376, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.28.attn_output.weight", "offset": 3606151168, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 3617685504, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.28.attn_v.weight", "offset": 3629219840, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 3630661632, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3630678016, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.29.ffn_gate.weight", "offset": 3661676544, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.29.ffn_up.weight", "offset": 3692675072, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.29.ffn_norm.weight", "offset": 3723673600, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 3723689984, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.29.attn_output.weight", "offset": 3725131776, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 3736666112, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.29.attn_v.weight", "offset": 3748200448, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 3749642240, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3749658624, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.30.ffn_gate.weight", "offset": 3780657152, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.30.ffn_up.weight", "offset": 3811655680, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.30.ffn_norm.weight", "offset": 3842654208, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 3842670592, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.30.attn_output.weight", "offset": 3844112384, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 3855646720, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.30.attn_v.weight", "offset": 3867181056, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 3868622848, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 3868639232, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.31.ffn_gate.weight", "offset": 3899637760, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.31.ffn_up.weight", "offset": 3930636288, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.31.ffn_norm.weight", "offset": 3961634816, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 3961651200, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.31.attn_output.weight", "offset": 3963092992, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.31.attn_q.weight", "offset": 3974627328, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.31.attn_v.weight", "offset": 3986161664, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 3987603456, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 3987619840, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.32.ffn_gate.weight", "offset": 4018618368, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.32.ffn_up.weight", "offset": 4049616896, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.32.ffn_norm.weight", "offset": 4080615424, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 4080631808, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.32.attn_output.weight", "offset": 4082073600, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.32.attn_q.weight", "offset": 4093607936, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.32.attn_v.weight", "offset": 4105142272, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.33.attn_norm.weight", "offset": 4106584064, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 4106600448, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.33.ffn_gate.weight", "offset": 4137598976, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.33.ffn_up.weight", "offset": 4168597504, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.33.ffn_norm.weight", "offset": 4199596032, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 4199612416, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.33.attn_output.weight", "offset": 4201054208, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.33.attn_q.weight", "offset": 4212588544, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.33.attn_v.weight", "offset": 4224122880, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 4225564672, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 4225581056, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.34.ffn_gate.weight", "offset": 4256579584, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.34.ffn_up.weight", "offset": 4287578112, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.34.ffn_norm.weight", "offset": 4318576640, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 4318593024, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.34.attn_output.weight", "offset": 4320034816, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.34.attn_q.weight", "offset": 4331569152, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.34.attn_v.weight", "offset": 4343103488, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.35.attn_norm.weight", "offset": 4344545280, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 4344561664, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.35.ffn_gate.weight", "offset": 4375560192, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.35.ffn_up.weight", "offset": 4406558720, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.35.ffn_norm.weight", "offset": 4437557248, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 4437573632, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.35.attn_output.weight", "offset": 4439015424, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.35.attn_q.weight", "offset": 4450549760, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.35.attn_v.weight", "offset": 4462084096, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 4463525888, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 4463542272, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 4494540800, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.36.ffn_up.weight", "offset": 4525539328, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.36.ffn_norm.weight", "offset": 4556537856, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 4556554240, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.36.attn_output.weight", "offset": 4557996032, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.36.attn_q.weight", "offset": 4569530368, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.36.attn_v.weight", "offset": 4581064704, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.37.attn_norm.weight", "offset": 4582506496, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 4582522880, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.37.ffn_gate.weight", "offset": 4613521408, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.37.ffn_up.weight", "offset": 4644519936, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.37.ffn_norm.weight", "offset": 4675518464, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 4675534848, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.37.attn_output.weight", "offset": 4676976640, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.37.attn_q.weight", "offset": 4688510976, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.37.attn_v.weight", "offset": 4700045312, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 4701487104, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 4701503488, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.38.ffn_gate.weight", "offset": 4732502016, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.38.ffn_up.weight", "offset": 4763500544, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.38.ffn_norm.weight", "offset": 4794499072, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 4794515456, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.38.attn_output.weight", "offset": 4795957248, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.38.attn_q.weight", "offset": 4807491584, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.38.attn_v.weight", "offset": 4819025920, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 4820467712, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 4820484096, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.39.ffn_gate.weight", "offset": 4851482624, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.39.ffn_up.weight", "offset": 4882481152, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.39.ffn_norm.weight", "offset": 4913479680, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 4913496064, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.39.attn_output.weight", "offset": 4914937856, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.39.attn_q.weight", "offset": 4926472192, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.39.attn_v.weight", "offset": 4938006528, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 4939448320, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 4939464704, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.40.ffn_gate.weight", "offset": 4970463232, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.40.ffn_up.weight", "offset": 5001461760, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.40.ffn_norm.weight", "offset": 5032460288, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 5032476672, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.40.attn_output.weight", "offset": 5033918464, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.40.attn_q.weight", "offset": 5045452800, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.40.attn_v.weight", "offset": 5056987136, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.41.ffn_gate.weight", "offset": 5058428928, "shape": [ 4096, 11008 ], "size": 1441792, "type": 13 }, { "name": "blk.41.attn_k.weight", "offset": 5089427456, "shape": [ 4096, 512 ], "size": 30998528, "type": 13 }, { "name": "blk.41.attn_output.weight", "offset": 5090869248, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.41.attn_q.weight", "offset": 5102403584, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.41.attn_v.weight", "offset": 5113937920, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "output.weight", "offset": 5115379712, "shape": [ 4096, 64000 ], "size": 1441792, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 5330419712, "shape": [ 4096 ], "size": 215040000, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 5330436096, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.41.ffn_up.weight", "offset": 5361434624, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.41.ffn_norm.weight", "offset": 5392433152, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 5392449536, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 5392465920, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.42.ffn_gate.weight", "offset": 5423464448, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.42.ffn_up.weight", "offset": 5454462976, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.42.ffn_norm.weight", "offset": 5485461504, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 5485477888, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.42.attn_output.weight", "offset": 5486919680, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.42.attn_q.weight", "offset": 5498454016, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.42.attn_v.weight", "offset": 5509988352, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 5511430144, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 5511446528, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.43.ffn_gate.weight", "offset": 5542445056, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.43.ffn_up.weight", "offset": 5573443584, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.43.ffn_norm.weight", "offset": 5604442112, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 5604458496, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.43.attn_output.weight", "offset": 5605900288, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.43.attn_q.weight", "offset": 5617434624, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.43.attn_v.weight", "offset": 5628968960, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.44.attn_norm.weight", "offset": 5630410752, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 5630427136, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.44.ffn_gate.weight", "offset": 5661425664, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.44.ffn_up.weight", "offset": 5692424192, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.44.ffn_norm.weight", "offset": 5723422720, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 5723439104, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.44.attn_output.weight", "offset": 5724880896, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.44.attn_q.weight", "offset": 5736415232, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.44.attn_v.weight", "offset": 5747949568, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 5749391360, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 5749407744, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.45.ffn_gate.weight", "offset": 5780406272, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.45.ffn_up.weight", "offset": 5811404800, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.45.ffn_norm.weight", "offset": 5842403328, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 5842419712, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.45.attn_output.weight", "offset": 5843861504, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.45.attn_q.weight", "offset": 5855395840, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.45.attn_v.weight", "offset": 5866930176, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.46.attn_norm.weight", "offset": 5868371968, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 5868388352, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.46.ffn_gate.weight", "offset": 5899386880, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.46.ffn_up.weight", "offset": 5930385408, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.46.ffn_norm.weight", "offset": 5961383936, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 5961400320, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.46.attn_output.weight", "offset": 5962842112, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.46.attn_q.weight", "offset": 5974376448, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.46.attn_v.weight", "offset": 5985910784, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 5987352576, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 5987368960, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.47.ffn_gate.weight", "offset": 6018367488, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.47.ffn_up.weight", "offset": 6049366016, "shape": [ 4096, 11008 ], "size": 30998528, "type": 13 }, { "name": "blk.47.ffn_norm.weight", "offset": 6080364544, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 6080380928, "shape": [ 4096, 512 ], "size": 16384, "type": 13 }, { "name": "blk.47.attn_output.weight", "offset": 6081822720, "shape": [ 4096, 4096 ], "size": 1441792, "type": 13 }, { "name": "blk.47.attn_q.weight", "offset": 6093357056, "shape": [ 4096, 4096 ], "size": 11534336, "type": 13 }, { "name": "blk.47.attn_v.weight", "offset": 6104891392, "shape": [ 4096, 512 ], "size": 11534336, "type": 13 }, { "name": "output_norm.weight", "offset": 6106333184, "shape": [ 4096 ], "size": 1441792, "type": 0 } ], "version": 3 }