orca-mini

{ "metadata": { "general.architecture": "llama", "general.file_type": 9, "general.name": "LLaMA v2", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 32, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 32, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32000 values)", "tokenizer.ggml.token_type": "... (32000 values)", "tokenizer.ggml.tokens": "... (32000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 6738415616, "tensors": [ { "name": "blk.0.attn_q.weight", "offset": 98304000, "shape": [ 4096, 4096 ], "size": 98304000, "type": 7 }, { "name": "blk.0.attn_k.weight", "offset": 110886912, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.0.attn_v.weight", "offset": 123469824, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.0.attn_output.weight", "offset": 136052736, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.0.ffn_gate.weight", "offset": 148635648, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 182452224, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.0.ffn_down.weight", "offset": 216268800, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.0.attn_norm.weight", "offset": 250085376, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 250101760, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 250118144, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.1.attn_k.weight", "offset": 262701056, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.1.attn_v.weight", "offset": 275283968, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.1.attn_output.weight", "offset": 287866880, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.1.ffn_gate.weight", "offset": 300449792, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.1.ffn_up.weight", "offset": 334266368, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.1.ffn_down.weight", "offset": 368082944, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 401899520, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 401915904, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 401932288, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.2.attn_k.weight", "offset": 414515200, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.2.attn_v.weight", "offset": 427098112, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.2.attn_output.weight", "offset": 439681024, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.2.ffn_gate.weight", "offset": 452263936, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.2.ffn_up.weight", "offset": 486080512, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.2.ffn_down.weight", "offset": 519897088, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 553713664, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 553730048, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 553746432, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.3.attn_k.weight", "offset": 566329344, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.3.attn_v.weight", "offset": 578912256, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.3.attn_output.weight", "offset": 591495168, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.3.ffn_gate.weight", "offset": 604078080, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.3.ffn_up.weight", "offset": 637894656, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.3.ffn_down.weight", "offset": 671711232, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 705527808, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 705544192, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 705560576, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.4.attn_k.weight", "offset": 718143488, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.4.attn_v.weight", "offset": 730726400, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.4.attn_output.weight", "offset": 743309312, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.4.ffn_gate.weight", "offset": 755892224, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.4.ffn_up.weight", "offset": 789708800, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.4.ffn_down.weight", "offset": 823525376, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 857341952, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 857358336, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 857374720, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.5.attn_k.weight", "offset": 869957632, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.5.attn_v.weight", "offset": 882540544, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.5.attn_output.weight", "offset": 895123456, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.5.ffn_gate.weight", "offset": 907706368, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.5.ffn_up.weight", "offset": 941522944, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.5.ffn_down.weight", "offset": 975339520, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 1009156096, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 1009172480, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1009188864, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.6.attn_k.weight", "offset": 1021771776, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.6.attn_v.weight", "offset": 1034354688, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 1046937600, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.6.ffn_gate.weight", "offset": 1059520512, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.6.ffn_up.weight", "offset": 1093337088, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.6.ffn_down.weight", "offset": 1127153664, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 1160970240, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 1160986624, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 1161003008, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.7.attn_k.weight", "offset": 1173585920, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.7.attn_v.weight", "offset": 1186168832, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.7.attn_output.weight", "offset": 1198751744, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.7.ffn_gate.weight", "offset": 1211334656, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 1245151232, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.7.ffn_down.weight", "offset": 1278967808, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.7.attn_norm.weight", "offset": 1312784384, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 1312800768, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 1312817152, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.8.attn_k.weight", "offset": 1325400064, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.8.attn_v.weight", "offset": 1337982976, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.8.attn_output.weight", "offset": 1350565888, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.8.ffn_gate.weight", "offset": 1363148800, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.8.ffn_up.weight", "offset": 1396965376, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.8.ffn_down.weight", "offset": 1430781952, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.8.attn_norm.weight", "offset": 1464598528, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 1464614912, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1464631296, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.9.attn_k.weight", "offset": 1477214208, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.9.attn_v.weight", "offset": 1489797120, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.9.attn_output.weight", "offset": 1502380032, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.9.ffn_gate.weight", "offset": 1514962944, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.9.ffn_up.weight", "offset": 1548779520, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.9.ffn_down.weight", "offset": 1582596096, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 1616412672, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "tn_k.weight\u0002\u0000\u0000\u0000\u0000\u0010\u0000\u0000\u0000\u0000", "offset": 1616429056, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1616445440, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.10.attn_k.weight", "offset": 1629028352, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.10.attn_v.weight", "offset": 1641611264, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.10.attn_output.weight", "offset": 1654194176, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.10.ffn_gate.weight", "offset": 1666777088, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.10.ffn_up.weight", "offset": 1700593664, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.10.ffn_down.weight", "offset": 1734410240, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 1768226816, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 1768243200, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1768259584, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.11.attn_k.weight", "offset": 1780842496, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.11.attn_v.weight", "offset": 1793425408, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.11.attn_output.weight", "offset": 1806008320, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.11.ffn_gate.weight", "offset": 1818591232, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.11.ffn_up.weight", "offset": 1852407808, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.11.ffn_down.weight", "offset": 1886224384, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 1920040960, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1920057344, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1920073728, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.12.attn_k.weight", "offset": 1932656640, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.12.attn_v.weight", "offset": 1945239552, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.12.attn_output.weight", "offset": 1957822464, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.12.ffn_gate.weight", "offset": 1970405376, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.12.ffn_up.weight", "offset": 2004221952, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.12.ffn_down.weight", "offset": 2038038528, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 2071855104, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 2071871488, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 2071887872, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.13.attn_k.weight", "offset": 2084470784, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.13.attn_v.weight", "offset": 2097053696, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.13.attn_output.weight", "offset": 2109636608, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.13.ffn_gate.weight", "offset": 2122219520, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.13.ffn_up.weight", "offset": 2156036096, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.13.ffn_down.weight", "offset": 2189852672, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 2223669248, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 2223685632, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 2223702016, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.14.attn_k.weight", "offset": 2236284928, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.14.attn_v.weight", "offset": 2248867840, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.14.attn_output.weight", "offset": 2261450752, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.14.ffn_gate.weight", "offset": 2274033664, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.14.ffn_up.weight", "offset": 2307850240, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.14.ffn_down.weight", "offset": 2341666816, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 2375483392, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 2375499776, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 2375516160, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.15.attn_k.weight", "offset": 2388099072, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.15.attn_v.weight", "offset": 2400681984, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.15.attn_output.weight", "offset": 2413264896, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.15.ffn_gate.weight", "offset": 2425847808, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.15.ffn_up.weight", "offset": 2459664384, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.15.ffn_down.weight", "offset": 2493480960, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 2527297536, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 2527313920, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 2527330304, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.16.attn_k.weight", "offset": 2539913216, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.16.attn_v.weight", "offset": 2552496128, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.16.attn_output.weight", "offset": 2565079040, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.16.ffn_gate.weight", "offset": 2577661952, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 2611478528, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.16.ffn_down.weight", "offset": 2645295104, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 2679111680, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 2679128064, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 2679144448, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.17.attn_k.weight", "offset": 2691727360, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.17.attn_v.weight", "offset": 2704310272, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.17.attn_output.weight", "offset": 2716893184, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.17.ffn_gate.weight", "offset": 2729476096, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.17.ffn_up.weight", "offset": 2763292672, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.17.ffn_down.weight", "offset": 2797109248, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.17.attn_norm.weight", "offset": 2830925824, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 2830942208, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2830958592, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.18.attn_k.weight", "offset": 2843541504, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.18.attn_v.weight", "offset": 2856124416, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.18.attn_output.weight", "offset": 2868707328, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.18.ffn_gate.weight", "offset": 2881290240, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.18.ffn_up.weight", "offset": 2915106816, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.18.ffn_down.weight", "offset": 2948923392, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 2982739968, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 2982756352, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 2982772736, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.19.attn_k.weight", "offset": 2995355648, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.19.attn_v.weight", "offset": 3007938560, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.19.attn_output.weight", "offset": 3020521472, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.19.ffn_gate.weight", "offset": 3033104384, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.19.ffn_up.weight", "offset": 3066920960, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.19.ffn_down.weight", "offset": 3100737536, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.19.attn_norm.weight", "offset": 3134554112, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 3134570496, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 3134586880, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.20.attn_k.weight", "offset": 3147169792, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.20.attn_v.weight", "offset": 3159752704, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.20.attn_output.weight", "offset": 3172335616, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.20.ffn_gate.weight", "offset": 3184918528, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.20.ffn_up.weight", "offset": 3218735104, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.20.ffn_down.weight", "offset": 3252551680, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 3286368256, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 3286384640, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 3286401024, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.21.attn_k.weight", "offset": 3298983936, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.21.attn_v.weight", "offset": 3311566848, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 3324149760, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.21.ffn_gate.weight", "offset": 3336732672, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.21.ffn_up.weight", "offset": 3370549248, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.21.ffn_down.weight", "offset": 3404365824, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 3438182400, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 3438198784, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 3438215168, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.22.attn_k.weight", "offset": 3450798080, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.22.attn_v.weight", "offset": 3463380992, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.22.attn_output.weight", "offset": 3475963904, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.22.ffn_gate.weight", "offset": 3488546816, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.22.ffn_up.weight", "offset": 3522363392, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.22.ffn_down.weight", "offset": 3556179968, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.22.attn_norm.weight", "offset": 3589996544, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 3590012928, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 3590029312, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.23.attn_k.weight", "offset": 3602612224, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.23.attn_v.weight", "offset": 3615195136, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.23.attn_output.weight", "offset": 3627778048, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.23.ffn_gate.weight", "offset": 3640360960, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.23.ffn_up.weight", "offset": 3674177536, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.23.ffn_down.weight", "offset": 3707994112, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 3741810688, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 3741827072, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 3741843456, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.24.attn_k.weight", "offset": 3754426368, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.24.attn_v.weight", "offset": 3767009280, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.24.attn_output.weight", "offset": 3779592192, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.24.ffn_gate.weight", "offset": 3792175104, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.24.ffn_up.weight", "offset": 3825991680, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.24.ffn_down.weight", "offset": 3859808256, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.24.attn_norm.weight", "offset": 3893624832, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 3893641216, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 3893657600, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "\ufffd\ufffd\ufffd\ufffd\t\ufffdy\ufffd\ufffd\ufffd\ufffdmgo{\ufffd블\ufffd", "offset": 3906240512, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.25.attn_v.weight", "offset": 3918823424, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.25.attn_output.weight", "offset": 3931406336, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.25.ffn_gate.weight", "offset": 3943989248, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.25.ffn_up.weight", "offset": 3977805824, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.25.ffn_down.weight", "offset": 4011622400, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.25.attn_norm.weight", "offset": 4045438976, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 4045455360, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 4045471744, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.26.attn_k.weight", "offset": 4058054656, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.26.attn_v.weight", "offset": 4070637568, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.26.attn_output.weight", "offset": 4083220480, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.26.ffn_gate.weight", "offset": 4095803392, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.26.ffn_up.weight", "offset": 4129619968, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.26.ffn_down.weight", "offset": 4163436544, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.26.attn_norm.weight", "offset": 4197253120, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 4197269504, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 4197285888, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.27.attn_k.weight", "offset": 4209868800, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.27.attn_v.weight", "offset": 4222451712, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.27.attn_output.weight", "offset": 4235034624, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.27.ffn_gate.weight", "offset": 4247617536, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.27.ffn_up.weight", "offset": 4281434112, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.27.ffn_down.weight", "offset": 4315250688, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.27.attn_norm.weight", "offset": 4349067264, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 4349083648, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 4349100032, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.28.attn_k.weight", "offset": 4361682944, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.28.attn_v.weight", "offset": 4374265856, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.28.attn_output.weight", "offset": 4386848768, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.28.ffn_gate.weight", "offset": 4399431680, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.28.ffn_up.weight", "offset": 4433248256, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.28.ffn_down.weight", "offset": 4467064832, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.28.attn_norm.weight", "offset": 4500881408, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 4500897792, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 4500914176, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.29.attn_k.weight", "offset": 4513497088, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.29.attn_v.weight", "offset": 4526080000, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.29.attn_output.weight", "offset": 4538662912, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.29.ffn_gate.weight", "offset": 4551245824, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.29.ffn_up.weight", "offset": 4585062400, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.29.ffn_down.weight", "offset": 4618878976, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.29.attn_norm.weight", "offset": 4652695552, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 4652711936, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 4652728320, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.30.attn_k.weight", "offset": 4665311232, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.30.attn_v.weight", "offset": 4677894144, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.30.attn_output.weight", "offset": 4690477056, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.30.ffn_gate.weight", "offset": 4703059968, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.30.ffn_up.weight", "offset": 4736876544, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.30.ffn_down.weight", "offset": 4770693120, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.30.attn_norm.weight", "offset": 4804509696, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 4804526080, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 4804542464, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.31.attn_k.weight", "offset": 4817125376, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.31.attn_v.weight", "offset": 4829708288, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.31.attn_output.weight", "offset": 4842291200, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.31.ffn_gate.weight", "offset": 4854874112, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.31.ffn_up.weight", "offset": 4888690688, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.31.ffn_down.weight", "offset": 4922507264, "shape": [ 11008, 4096 ], "size": 33816576, "type": 7 }, { "name": "blk.31.attn_norm.weight", "offset": 4956323840, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 4956340224, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "output_norm.weight", "offset": 4956356608, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "output.weight", "offset": 4956372992, "shape": [ 4096, 32000 ], "size": 16384, "type": 14 } ], "version": 2 }