vicuna

{ "metadata": { "general.architecture": "llama", "general.file_type": 14, "general.name": "lmsys", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 32, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 32, "llama.context_length": 2048, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.scores": "... (32000 values)", "tokenizer.ggml.token_type": "... (32000 values)", "tokenizer.ggml.tokens": "... (32000 values)" }, "num_params": 6738415616, "tensors": [ { "name": "blk.0.attn_q.weight", "offset": 73728000, "shape": [ 4096, 4096 ], "size": 73728000, "type": 12 }, { "name": "blk.0.attn_k.weight", "offset": 83165184, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.0.attn_v.weight", "offset": 92602368, "shape": [ 4096, 4096 ], "size": 9437184, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 104136704, "shape": [ 4096, 4096 ], "size": 11534336, "type": 12 }, { "name": "blk.0.ffn_gate.weight", "offset": 113573888, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.0.ffn_down.weight", "offset": 138936320, "shape": [ 11008, 4096 ], "size": 25362432, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 169934848, "shape": [ 4096, 11008 ], "size": 30998528, "type": 12 }, { "name": "blk.0.attn_norm.weight", "offset": 195297280, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 195313664, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 195330048, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.1.attn_k.weight", "offset": 204767232, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.1.attn_v.weight", "offset": 214204416, "shape": [ 4096, 4096 ], "size": 9437184, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 225738752, "shape": [ 4096, 4096 ], "size": 11534336, "type": 12 }, { "name": "blk.1.ffn_gate.weight", "offset": 235175936, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.1.ffn_down.weight", "offset": 260538368, "shape": [ 11008, 4096 ], "size": 25362432, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 291536896, "shape": [ 4096, 11008 ], "size": 30998528, "type": 12 }, { "name": "blk.1.attn_norm.weight", "offset": 316899328, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 316915712, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 316932096, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.2.attn_k.weight", "offset": 326369280, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.2.attn_v.weight", "offset": 335806464, "shape": [ 4096, 4096 ], "size": 9437184, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 347340800, "shape": [ 4096, 4096 ], "size": 11534336, "type": 12 }, { "name": "blk.2.ffn_gate.weight", "offset": 356777984, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.2.ffn_down.weight", "offset": 382140416, "shape": [ 11008, 4096 ], "size": 25362432, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 413138944, "shape": [ 4096, 11008 ], "size": 30998528, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 438501376, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 438517760, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 438534144, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.3.attn_k.weight", "offset": 447971328, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.3.attn_v.weight", "offset": 457408512, "shape": [ 4096, 4096 ], "size": 9437184, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 468942848, "shape": [ 4096, 4096 ], "size": 11534336, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 478380032, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.3.ffn_down.weight", "offset": 503742464, "shape": [ 11008, 4096 ], "size": 25362432, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 534740992, "shape": [ 4096, 11008 ], "size": 30998528, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 560103424, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 560119808, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 560136192, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.4.attn_k.weight", "offset": 569573376, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.4.attn_v.weight", "offset": 579010560, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 588447744, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 597884928, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.4.ffn_down.weight", "offset": 623247360, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 648609792, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 673972224, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 673988608, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 674004992, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.5.attn_k.weight", "offset": 683442176, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.5.attn_v.weight", "offset": 692879360, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 702316544, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 711753728, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.5.ffn_down.weight", "offset": 737116160, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 762478592, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 787841024, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 787857408, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 787873792, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.6.attn_k.weight", "offset": 797310976, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.6.attn_v.weight", "offset": 806748160, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 816185344, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 825622528, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.6.ffn_down.weight", "offset": 850984960, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 876347392, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 901709824, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 901726208, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 901742592, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.7.attn_k.weight", "offset": 911179776, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.7.attn_v.weight", "offset": 920616960, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 930054144, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 939491328, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.7.ffn_down.weight", "offset": 964853760, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 990216192, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 1015578624, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 1015595008, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 1015611392, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.8.attn_k.weight", "offset": 1025048576, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.8.attn_v.weight", "offset": 1034485760, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 1043922944, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 1053360128, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.8.ffn_down.weight", "offset": 1078722560, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 1104084992, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 1129447424, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 1129463808, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1129480192, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.9.attn_k.weight", "offset": 1138917376, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.9.attn_v.weight", "offset": 1148354560, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 1157791744, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 1167228928, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.9.ffn_down.weight", "offset": 1192591360, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 1217953792, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 1243316224, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "tn_k.weight\u0002\u0000\u0000\u0000\u0000\u0010\u0000\u0000\u0000\u0000", "offset": 1243332608, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1243348992, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.10.attn_k.weight", "offset": 1252786176, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.10.attn_v.weight", "offset": 1262223360, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 1271660544, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.10.ffn_gate.weight", "offset": 1281097728, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.10.ffn_down.weight", "offset": 1306460160, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 1331822592, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1357185024, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 1357201408, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1357217792, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.11.attn_k.weight", "offset": 1366654976, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.11.attn_v.weight", "offset": 1376092160, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 1385529344, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 1394966528, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.11.ffn_down.weight", "offset": 1420328960, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 1445691392, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1471053824, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1471070208, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1471086592, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.12.attn_k.weight", "offset": 1480523776, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.12.attn_v.weight", "offset": 1489960960, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 1499398144, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 1508835328, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.12.ffn_down.weight", "offset": 1534197760, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 1559560192, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1584922624, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 1584939008, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1584955392, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.13.attn_k.weight", "offset": 1594392576, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.13.attn_v.weight", "offset": 1603829760, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 1613266944, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 1622704128, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.13.ffn_down.weight", "offset": 1648066560, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 1673428992, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 1698791424, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 1698807808, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 1698824192, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.14.attn_k.weight", "offset": 1708261376, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.14.attn_v.weight", "offset": 1717698560, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 1727135744, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 1736572928, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.14.ffn_down.weight", "offset": 1761935360, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 1787297792, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 1812660224, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 1812676608, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 1812692992, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.15.attn_k.weight", "offset": 1822130176, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.15.attn_v.weight", "offset": 1831567360, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 1841004544, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 1850441728, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.15.ffn_down.weight", "offset": 1875804160, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 1901166592, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 1926529024, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 1926545408, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1926561792, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.16.attn_k.weight", "offset": 1935998976, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.16.attn_v.weight", "offset": 1945436160, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 1954873344, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 1964310528, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.16.ffn_down.weight", "offset": 1989672960, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 2015035392, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 2040397824, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 2040414208, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 2040430592, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.17.attn_k.weight", "offset": 2049867776, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.17.attn_v.weight", "offset": 2059304960, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 2068742144, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 2078179328, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.17.ffn_down.weight", "offset": 2103541760, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 2128904192, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 2154266624, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 2154283008, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2154299392, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.18.attn_k.weight", "offset": 2163736576, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.18.attn_v.weight", "offset": 2173173760, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 2182610944, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 2192048128, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.18.ffn_down.weight", "offset": 2217410560, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 2242772992, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 2268135424, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 2268151808, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 2268168192, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.19.attn_k.weight", "offset": 2277605376, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.19.attn_v.weight", "offset": 2287042560, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 2296479744, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 2305916928, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.19.ffn_down.weight", "offset": 2331279360, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 2356641792, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 2382004224, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 2382020608, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 2382036992, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.20.attn_k.weight", "offset": 2391474176, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.20.attn_v.weight", "offset": 2400911360, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 2410348544, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 2419785728, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.20.ffn_down.weight", "offset": 2445148160, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 2470510592, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 2495873024, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 2495889408, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 2495905792, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.21.attn_k.weight", "offset": 2505342976, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.21.attn_v.weight", "offset": 2514780160, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 2524217344, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 2533654528, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.21.ffn_down.weight", "offset": 2559016960, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 2584379392, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 2609741824, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 2609758208, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 2609774592, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.22.attn_k.weight", "offset": 2619211776, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.22.attn_v.weight", "offset": 2628648960, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 2638086144, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 2647523328, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.22.ffn_down.weight", "offset": 2672885760, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 2698248192, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 2723610624, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 2723627008, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 2723643392, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.23.attn_k.weight", "offset": 2733080576, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.23.attn_v.weight", "offset": 2742517760, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 2751954944, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 2761392128, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.23.ffn_down.weight", "offset": 2786754560, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 2812116992, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 2837479424, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 2837495808, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 2837512192, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.24.attn_k.weight", "offset": 2846949376, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.24.attn_v.weight", "offset": 2856386560, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 2865823744, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 2875260928, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.24.ffn_down.weight", "offset": 2900623360, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 2925985792, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 2951348224, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 2951364608, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 2951380992, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "\u000b\ufffd\ufffd}\ufffd쌍\ufffd\ufffd\ufffd:zKp\ufffdO+?\ufffd", "offset": 2960818176, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.25.attn_v.weight", "offset": 2970255360, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 2979692544, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 2989129728, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.25.ffn_down.weight", "offset": 3014492160, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 3039854592, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 3065217024, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 3065233408, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 3065249792, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.26.attn_k.weight", "offset": 3074686976, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.26.attn_v.weight", "offset": 3084124160, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 3093561344, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 3102998528, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.26.ffn_down.weight", "offset": 3128360960, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 3153723392, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 3179085824, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 3179102208, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 3179118592, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.27.attn_k.weight", "offset": 3188555776, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.27.attn_v.weight", "offset": 3197992960, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 3207430144, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 3216867328, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.27.ffn_down.weight", "offset": 3242229760, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 3267592192, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 3292954624, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 3292971008, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 3292987392, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.28.attn_k.weight", "offset": 3302424576, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.28.attn_v.weight", "offset": 3311861760, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 3321298944, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.28.ffn_gate.weight", "offset": 3330736128, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.28.ffn_down.weight", "offset": 3356098560, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 3381460992, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 3406823424, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 3406839808, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 3406856192, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.29.attn_k.weight", "offset": 3416293376, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.29.attn_v.weight", "offset": 3425730560, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 3435167744, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.29.ffn_gate.weight", "offset": 3444604928, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.29.ffn_down.weight", "offset": 3469967360, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 3495329792, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 3520692224, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 3520708608, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 3520724992, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.30.attn_k.weight", "offset": 3530162176, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.30.attn_v.weight", "offset": 3539599360, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 3549036544, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.30.ffn_gate.weight", "offset": 3558473728, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.30.ffn_down.weight", "offset": 3583836160, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 3609198592, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 3634561024, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 3634577408, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 3634593792, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.31.attn_k.weight", "offset": 3644030976, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.31.attn_v.weight", "offset": 3653468160, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.31.attn_output.weight", "offset": 3662905344, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.31.ffn_gate.weight", "offset": 3672342528, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.31.ffn_down.weight", "offset": 3697704960, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 3723067392, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 3748429824, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 3748446208, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "output_norm.weight", "offset": 3748462592, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "output.weight", "offset": 3748478976, "shape": [ 4096, 32000 ], "size": 16384, "type": 14 } ], "version": 2 }