yi

{ "metadata": { "general.architecture": "llama", "general.file_type": 15, "general.name": "Yi-1.5-9B", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 4, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 48, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 5000000, "llama.vocab_size": 64000, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (64000 values)", "tokenizer.ggml.token_type": "... (64000 values)", "tokenizer.ggml.tokens": "... (64000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 8829407232, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 147456000, "shape": [ 4096 ], "size": 147456000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 147472384, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 184459264, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 209821696, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.0.ffn_norm.weight", "offset": 235184128, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 235200512, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 236380160, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 245817344, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.0.attn_v.weight", "offset": 255254528, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 256974848, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 256991232, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 293978112, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 319340544, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 344702976, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 344719360, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 345899008, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 355336192, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.1.attn_v.weight", "offset": 364773376, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 366493696, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 366510080, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 403496960, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 428859392, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 454221824, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 454238208, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 455417856, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.10.attn_q.weight", "offset": 464855040, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.10.attn_v.weight", "offset": 474292224, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 476012544, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 476028928, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.11.ffn_gate.weight", "offset": 513015808, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 538378240, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 563740672, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 563757056, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 564936704, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.11.attn_q.weight", "offset": 574373888, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.11.attn_v.weight", "offset": 583811072, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.12.ffn_gate.weight", "offset": 585531392, "shape": [ 4096, 11008 ], "size": 1720320, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 610893824, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.12.attn_k.weight", "offset": 636256256, "shape": [ 4096, 512 ], "size": 25362432, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 637435904, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.12.attn_q.weight", "offset": 646873088, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.12.attn_v.weight", "offset": 656310272, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 658030592, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 658046976, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 695033856, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 720396288, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 745758720, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 745775104, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 746954752, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 756391936, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.2.attn_v.weight", "offset": 765829120, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 767549440, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 767565824, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 804552704, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 829915136, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 855277568, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 855293952, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 856473600, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 865910784, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.3.attn_v.weight", "offset": 875347968, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 876527616, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 876544000, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 901906432, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 927268864, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 952631296, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 952647680, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 953827328, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 963264512, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.4.attn_v.weight", "offset": 972701696, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 973881344, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 973897728, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 999260160, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 1024622592, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 1049985024, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1050001408, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 1051181056, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 1060618240, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.5.attn_v.weight", "offset": 1070055424, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 1071775744, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1071792128, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.6.ffn_gate.weight", "offset": 1108779008, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 1134141440, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 1159503872, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1159520256, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 1160699904, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 1170137088, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.6.attn_v.weight", "offset": 1179574272, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 1180753920, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1180770304, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 1206132736, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 1231495168, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 1256857600, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1256873984, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 1258053632, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 1267490816, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.7.attn_v.weight", "offset": 1276928000, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 1278107648, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1278124032, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 1303486464, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 1328848896, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 1354211328, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1354227712, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 1355407360, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 1364844544, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.8.attn_v.weight", "offset": 1374281728, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 1376002048, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1376018432, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 1413005312, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 1438367744, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.9.ffn_norm.weight", "offset": 1463730176, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1463746560, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 1464926208, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 1474363392, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.9.attn_v.weight", "offset": 1483800576, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1484980224, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1484996608, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 1510359040, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 1510375424, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1510391808, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 1535754240, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 1561116672, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 1586479104, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1586495488, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 1587675136, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.13.attn_q.weight", "offset": 1597112320, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.13.attn_v.weight", "offset": 1606549504, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 1607729152, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1607745536, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.14.ffn_gate.weight", "offset": 1644732416, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 1670094848, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 1695457280, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1695473664, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 1696653312, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 1706090496, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.14.attn_v.weight", "offset": 1715527680, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.15.attn_norm.weight", "offset": 1717248000, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1717264384, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 1742626816, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 1767989248, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 1793351680, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1793368064, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 1794547712, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 1803984896, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.15.attn_v.weight", "offset": 1813422080, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 1814601728, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 1814618112, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 1839980544, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 1865342976, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 1890705408, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1890721792, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 1891901440, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.16.attn_q.weight", "offset": 1901338624, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.16.attn_v.weight", "offset": 1910775808, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 1911955456, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 1911971840, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.17.ffn_gate.weight", "offset": 1948958720, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 1974321152, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 1999683584, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 1999699968, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 2000879616, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 2010316800, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.17.attn_v.weight", "offset": 2019753984, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 2021474304, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2021490688, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 2046853120, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 2072215552, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 2097577984, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2097594368, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 2098774016, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.18.attn_q.weight", "offset": 2108211200, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.18.attn_v.weight", "offset": 2117648384, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 2118828032, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2118844416, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 2144206848, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 2169569280, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.19.ffn_norm.weight", "offset": 2194931712, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2194948096, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 2196127744, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 2205564928, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.19.attn_v.weight", "offset": 2215002112, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 2216181760, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2216198144, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.20.ffn_gate.weight", "offset": 2253185024, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 2278547456, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 2303909888, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 2303926272, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 2305105920, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 2314543104, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.20.attn_v.weight", "offset": 2323980288, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 2325700608, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2325716992, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 2351079424, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 2376441856, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 2401804288, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2401820672, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 2403000320, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 2412437504, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.21.attn_v.weight", "offset": 2421874688, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 2423054336, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2423070720, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 2448433152, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 2473795584, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 2499158016, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2499174400, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 2500354048, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 2509791232, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.22.attn_v.weight", "offset": 2519228416, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 2520408064, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2520424448, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.23.ffn_gate.weight", "offset": 2557411328, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 2582773760, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 2608136192, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2608152576, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 2609332224, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 2618769408, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.23.attn_v.weight", "offset": 2628206592, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 2629926912, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2629943296, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 2655305728, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 2680668160, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 2706030592, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 2706046976, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 2707226624, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 2716663808, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.24.attn_v.weight", "offset": 2726100992, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 2727280640, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 2727297024, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 2752659456, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 2778021888, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 2803384320, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 2803400704, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 2804580352, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 2814017536, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.25.attn_v.weight", "offset": 2823454720, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 2824634368, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 2824650752, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.26.ffn_gate.weight", "offset": 2861637632, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 2887000064, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 2912362496, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 2912378880, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 2913558528, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 2922995712, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.26.attn_v.weight", "offset": 2932432896, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.27.attn_k.weight", "offset": 2934153216, "shape": [ 4096, 512 ], "size": 1720320, "type": 12 }, { "name": "blk.27.attn_q.weight", "offset": 2935332864, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.27.attn_v.weight", "offset": 2944770048, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 2945949696, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 2945966080, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 2971328512, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 2996690944, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.27.ffn_norm.weight", "offset": 3022053376, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 3022069760, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 3031506944, "shape": [ 4096 ], "size": 9437184, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3031523328, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.28.ffn_gate.weight", "offset": 3056885760, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 3082248192, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.28.ffn_norm.weight", "offset": 3107610624, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3107627008, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 3108806656, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.28.attn_q.weight", "offset": 3118243840, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.28.attn_v.weight", "offset": 3127681024, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 3128860672, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3128877056, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.29.ffn_gate.weight", "offset": 3165863936, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 3191226368, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.29.ffn_norm.weight", "offset": 3216588800, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 3216605184, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 3217784832, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.29.attn_q.weight", "offset": 3227222016, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.29.attn_v.weight", "offset": 3236659200, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.30.attn_norm.weight", "offset": 3238379520, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3238395904, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.30.ffn_gate.weight", "offset": 3263758336, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 3289120768, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.30.ffn_norm.weight", "offset": 3314483200, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 3314499584, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 3315679232, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.30.attn_q.weight", "offset": 3325116416, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.30.attn_v.weight", "offset": 3334553600, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 3335733248, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 3335749632, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.31.ffn_gate.weight", "offset": 3361112064, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 3386474496, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.31.ffn_norm.weight", "offset": 3411836928, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 3411853312, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.31.attn_output.weight", "offset": 3413032960, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.31.attn_q.weight", "offset": 3422470144, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.31.attn_v.weight", "offset": 3431907328, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 3433086976, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 3433103360, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.32.ffn_gate.weight", "offset": 3470090240, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 3495452672, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.32.ffn_norm.weight", "offset": 3520815104, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 3520831488, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.32.attn_output.weight", "offset": 3522011136, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.32.attn_q.weight", "offset": 3531448320, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.32.attn_v.weight", "offset": 3540885504, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.33.attn_norm.weight", "offset": 3542605824, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 3542622208, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.33.ffn_gate.weight", "offset": 3567984640, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 3593347072, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.33.ffn_norm.weight", "offset": 3618709504, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 3618725888, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.33.attn_output.weight", "offset": 3619905536, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.33.attn_q.weight", "offset": 3629342720, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.33.attn_v.weight", "offset": 3638779904, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 3639959552, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 3639975936, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.34.ffn_gate.weight", "offset": 3665338368, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.34.ffn_up.weight", "offset": 3690700800, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.34.ffn_norm.weight", "offset": 3716063232, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 3716079616, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.34.attn_output.weight", "offset": 3717259264, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.34.attn_q.weight", "offset": 3726696448, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.34.attn_v.weight", "offset": 3736133632, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.35.attn_norm.weight", "offset": 3737313280, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 3737329664, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.35.ffn_gate.weight", "offset": 3774316544, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 3799678976, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.35.ffn_norm.weight", "offset": 3825041408, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 3825057792, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 3826237440, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.35.attn_q.weight", "offset": 3835674624, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.35.attn_v.weight", "offset": 3845111808, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.36.attn_norm.weight", "offset": 3846832128, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 3846848512, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.36.ffn_gate.weight", "offset": 3872210944, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 3897573376, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.36.ffn_norm.weight", "offset": 3922935808, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 3922952192, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.36.attn_output.weight", "offset": 3924131840, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.36.attn_q.weight", "offset": 3933569024, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.36.attn_v.weight", "offset": 3943006208, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 3944185856, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 3944202240, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.37.ffn_gate.weight", "offset": 3969564672, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 3994927104, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.37.ffn_norm.weight", "offset": 4020289536, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 4020305920, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.37.attn_output.weight", "offset": 4021485568, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.37.attn_q.weight", "offset": 4030922752, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.37.attn_v.weight", "offset": 4040359936, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 4041539584, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 4041555968, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.38.ffn_gate.weight", "offset": 4078542848, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 4103905280, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.38.ffn_norm.weight", "offset": 4129267712, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 4129284096, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.38.attn_output.weight", "offset": 4130463744, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.38.attn_q.weight", "offset": 4139900928, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.38.attn_v.weight", "offset": 4149338112, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.39.attn_norm.weight", "offset": 4151058432, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 4151074816, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.39.ffn_gate.weight", "offset": 4176437248, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 4201799680, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.39.ffn_norm.weight", "offset": 4227162112, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 4227178496, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.39.attn_output.weight", "offset": 4228358144, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.39.attn_q.weight", "offset": 4237795328, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.39.attn_v.weight", "offset": 4247232512, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.40.attn_norm.weight", "offset": 4248412160, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 4248428544, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.40.ffn_gate.weight", "offset": 4273790976, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.40.ffn_up.weight", "offset": 4299153408, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.40.ffn_norm.weight", "offset": 4324515840, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 4324532224, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.40.attn_output.weight", "offset": 4325711872, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.40.attn_q.weight", "offset": 4335149056, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.40.attn_v.weight", "offset": 4344586240, "shape": [ 4096, 512 ], "size": 9437184, "type": 12 }, { "name": "blk.41.ffn_gate.weight", "offset": 4345765888, "shape": [ 4096, 11008 ], "size": 1179648, "type": 12 }, { "name": "blk.41.attn_k.weight", "offset": 4371128320, "shape": [ 4096, 512 ], "size": 25362432, "type": 12 }, { "name": "blk.41.attn_output.weight", "offset": 4372307968, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.41.attn_q.weight", "offset": 4381745152, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.41.attn_v.weight", "offset": 4391182336, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "output.weight", "offset": 4392902656, "shape": [ 4096, 64000 ], "size": 1720320, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 4607942656, "shape": [ 4096 ], "size": 215040000, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 4607959040, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.41.ffn_up.weight", "offset": 4644945920, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.41.ffn_norm.weight", "offset": 4670308352, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 4670324736, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 4670341120, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.42.ffn_gate.weight", "offset": 4707328000, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.42.ffn_up.weight", "offset": 4732690432, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.42.ffn_norm.weight", "offset": 4758052864, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 4758069248, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.42.attn_output.weight", "offset": 4759248896, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.42.attn_q.weight", "offset": 4768686080, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.42.attn_v.weight", "offset": 4778123264, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.43.attn_norm.weight", "offset": 4779843584, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 4779859968, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.43.ffn_gate.weight", "offset": 4816846848, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.43.ffn_up.weight", "offset": 4842209280, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.43.ffn_norm.weight", "offset": 4867571712, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 4867588096, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.43.attn_output.weight", "offset": 4868767744, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.43.attn_q.weight", "offset": 4878204928, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.43.attn_v.weight", "offset": 4887642112, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.44.attn_norm.weight", "offset": 4889362432, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 4889378816, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.44.ffn_gate.weight", "offset": 4926365696, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.44.ffn_up.weight", "offset": 4951728128, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.44.ffn_norm.weight", "offset": 4977090560, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 4977106944, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.44.attn_output.weight", "offset": 4978286592, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.44.attn_q.weight", "offset": 4987723776, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.44.attn_v.weight", "offset": 4997160960, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.45.attn_norm.weight", "offset": 4998881280, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 4998897664, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.45.ffn_gate.weight", "offset": 5035884544, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.45.ffn_up.weight", "offset": 5061246976, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.45.ffn_norm.weight", "offset": 5086609408, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 5086625792, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.45.attn_output.weight", "offset": 5087805440, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.45.attn_q.weight", "offset": 5097242624, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.45.attn_v.weight", "offset": 5106679808, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.46.attn_norm.weight", "offset": 5108400128, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 5108416512, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.46.ffn_gate.weight", "offset": 5145403392, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.46.ffn_up.weight", "offset": 5170765824, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.46.ffn_norm.weight", "offset": 5196128256, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 5196144640, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.46.attn_output.weight", "offset": 5197324288, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.46.attn_q.weight", "offset": 5206761472, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.46.attn_v.weight", "offset": 5216198656, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "blk.47.attn_norm.weight", "offset": 5217918976, "shape": [ 4096 ], "size": 1720320, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 5217935360, "shape": [ 11008, 4096 ], "size": 16384, "type": 14 }, { "name": "blk.47.ffn_gate.weight", "offset": 5254922240, "shape": [ 4096, 11008 ], "size": 36986880, "type": 12 }, { "name": "blk.47.ffn_up.weight", "offset": 5280284672, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.47.ffn_norm.weight", "offset": 5305647104, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 5305663488, "shape": [ 4096, 512 ], "size": 16384, "type": 12 }, { "name": "blk.47.attn_output.weight", "offset": 5306843136, "shape": [ 4096, 4096 ], "size": 1179648, "type": 12 }, { "name": "blk.47.attn_q.weight", "offset": 5316280320, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.47.attn_v.weight", "offset": 5325717504, "shape": [ 4096, 512 ], "size": 9437184, "type": 14 }, { "name": "output_norm.weight", "offset": 5327437824, "shape": [ 4096 ], "size": 1720320, "type": 0 } ], "version": 3 }