nous-hermes

{ "metadata": { "general.architecture": "llama", "general.file_type": 12, "general.name": "LLaMA v2", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 32, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 32, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32000 values)", "tokenizer.ggml.token_type": "... (32000 values)", "tokenizer.ggml.tokens": "... (32000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 6738415616, "tensors": [ { "name": "blk.0.attn_q.weight", "offset": 56320000, "shape": [ 4096, 4096 ], "size": 56320000, "type": 11 }, { "name": "blk.0.attn_k.weight", "offset": 63528960, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 70737920, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 82272256, "shape": [ 4096, 4096 ], "size": 11534336, "type": 12 }, { "name": "blk.0.ffn_gate.weight", "offset": 91709440, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 111083520, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.0.ffn_down.weight", "offset": 130457600, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 161456128, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 161472512, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 161488896, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.1.attn_k.weight", "offset": 168697856, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 175906816, "shape": [ 4096, 4096 ], "size": 7208960, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 187441152, "shape": [ 4096, 4096 ], "size": 11534336, "type": 12 }, { "name": "blk.1.ffn_gate.weight", "offset": 196878336, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 216252416, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.1.ffn_down.weight", "offset": 235626496, "shape": [ 11008, 4096 ], "size": 19374080, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 266625024, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 266641408, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 266657792, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.2.attn_k.weight", "offset": 273866752, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.2.attn_v.weight", "offset": 281075712, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 290512896, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.2.ffn_gate.weight", "offset": 299950080, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 319324160, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.2.ffn_down.weight", "offset": 338698240, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 364060672, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 364077056, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 364093440, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.3.attn_k.weight", "offset": 371302400, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.3.attn_v.weight", "offset": 378511360, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 387948544, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 397385728, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 416759808, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.3.ffn_down.weight", "offset": 436133888, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 461496320, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 461512704, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 461529088, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.4.attn_k.weight", "offset": 468738048, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 475947008, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 485384192, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 494821376, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 514195456, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.4.ffn_down.weight", "offset": 533569536, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 558931968, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 558948352, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 558964736, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.5.attn_k.weight", "offset": 566173696, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 573382656, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 582819840, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 592257024, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 611631104, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.5.ffn_down.weight", "offset": 631005184, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 656367616, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 656384000, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 656400384, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.6.attn_k.weight", "offset": 663609344, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 670818304, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 680255488, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 689692672, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 709066752, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.6.ffn_down.weight", "offset": 728440832, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 753803264, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 753819648, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 753836032, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.7.attn_k.weight", "offset": 761044992, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 768253952, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 777691136, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 787128320, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 806502400, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.7.ffn_down.weight", "offset": 825876480, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 851238912, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 851255296, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 851271680, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.8.attn_k.weight", "offset": 858480640, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 865689600, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 875126784, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 884563968, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 903938048, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.8.ffn_down.weight", "offset": 923312128, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 948674560, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 948690944, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 948707328, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.9.attn_k.weight", "offset": 955916288, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.9.attn_v.weight", "offset": 963125248, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 972562432, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 981999616, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 1001373696, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.9.ffn_down.weight", "offset": 1020747776, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 1046110208, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "tn_k.weight\u0002\u0000\u0000\u0000\u0000\u0010\u0000\u0000\u0000\u0000", "offset": 1046126592, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1046142976, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.10.attn_k.weight", "offset": 1053351936, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 1060560896, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 1069998080, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.10.ffn_gate.weight", "offset": 1079435264, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 1098809344, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.10.ffn_down.weight", "offset": 1118183424, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1143545856, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 1143562240, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1143578624, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.11.attn_k.weight", "offset": 1150787584, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.11.attn_v.weight", "offset": 1157996544, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 1167433728, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 1176870912, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1196244992, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.11.ffn_down.weight", "offset": 1215619072, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1240981504, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1240997888, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1241014272, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.12.attn_k.weight", "offset": 1248223232, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.12.attn_v.weight", "offset": 1255432192, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 1264869376, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 1274306560, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 1293680640, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.12.ffn_down.weight", "offset": 1313054720, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1338417152, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 1338433536, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1338449920, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.13.attn_k.weight", "offset": 1345658880, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.13.attn_v.weight", "offset": 1352867840, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 1362305024, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 1371742208, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 1391116288, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.13.ffn_down.weight", "offset": 1410490368, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 1435852800, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 1435869184, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 1435885568, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.14.attn_k.weight", "offset": 1443094528, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 1450303488, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 1459740672, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 1469177856, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 1488551936, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.14.ffn_down.weight", "offset": 1507926016, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 1533288448, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 1533304832, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 1533321216, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.15.attn_k.weight", "offset": 1540530176, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 1547739136, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 1557176320, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 1566613504, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 1585987584, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.15.ffn_down.weight", "offset": 1605361664, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 1630724096, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 1630740480, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1630756864, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.16.attn_k.weight", "offset": 1637965824, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 1645174784, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 1654611968, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 1664049152, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 1683423232, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.16.ffn_down.weight", "offset": 1702797312, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 1728159744, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 1728176128, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 1728192512, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.17.attn_k.weight", "offset": 1735401472, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.17.attn_v.weight", "offset": 1742610432, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 1752047616, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 1761484800, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 1780858880, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.17.ffn_down.weight", "offset": 1800232960, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 1825595392, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 1825611776, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 1825628160, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.18.attn_k.weight", "offset": 1832837120, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 1840046080, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 1849483264, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 1858920448, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 1878294528, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.18.ffn_down.weight", "offset": 1897668608, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 1923031040, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 1923047424, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 1923063808, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.19.attn_k.weight", "offset": 1930272768, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 1937481728, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 1946918912, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 1956356096, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 1975730176, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.19.ffn_down.weight", "offset": 1995104256, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 2020466688, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 2020483072, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 2020499456, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.20.attn_k.weight", "offset": 2027708416, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 2034917376, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 2044354560, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 2053791744, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 2073165824, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.20.ffn_down.weight", "offset": 2092539904, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 2117902336, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 2117918720, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 2117935104, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.21.attn_k.weight", "offset": 2125144064, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 2132353024, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 2141790208, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 2151227392, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 2170601472, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.21.ffn_down.weight", "offset": 2189975552, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 2215337984, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 2215354368, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 2215370752, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.22.attn_k.weight", "offset": 2222579712, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 2229788672, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 2239225856, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 2248663040, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 2268037120, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.22.ffn_down.weight", "offset": 2287411200, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 2312773632, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 2312790016, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 2312806400, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.23.attn_k.weight", "offset": 2320015360, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 2327224320, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 2336661504, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 2346098688, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 2365472768, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.23.ffn_down.weight", "offset": 2384846848, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 2410209280, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 2410225664, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 2410242048, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.24.attn_k.weight", "offset": 2417451008, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 2424659968, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 2434097152, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 2443534336, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 2462908416, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.24.ffn_down.weight", "offset": 2482282496, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 2507644928, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 2507661312, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 2507677696, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "'U\u0004\ufffd(\ufffd\ufffd\ufffd\ufffd\u00017\ufffd :0\u00062^P\u001c", "offset": 2514886656, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 2522095616, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 2531532800, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 2540969984, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 2560344064, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.25.ffn_down.weight", "offset": 2579718144, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 2605080576, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 2605096960, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 2605113344, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.26.attn_k.weight", "offset": 2612322304, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 2619531264, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 2628968448, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 2638405632, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 2657779712, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.26.ffn_down.weight", "offset": 2677153792, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 2702516224, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 2702532608, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 2702548992, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.27.attn_k.weight", "offset": 2709757952, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 2716966912, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 2726404096, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 2735841280, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 2755215360, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.27.ffn_down.weight", "offset": 2774589440, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 2799951872, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 2799968256, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 2799984640, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.28.attn_k.weight", "offset": 2807193600, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 2814402560, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 2823839744, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.28.ffn_gate.weight", "offset": 2833276928, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 2852651008, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.28.ffn_down.weight", "offset": 2872025088, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 2897387520, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 2897403904, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 2897420288, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.29.attn_k.weight", "offset": 2904629248, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 2911838208, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 2921275392, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.29.ffn_gate.weight", "offset": 2930712576, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 2950086656, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.29.ffn_down.weight", "offset": 2969460736, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 2994823168, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 2994839552, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 2994855936, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.30.attn_k.weight", "offset": 3002064896, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 3009273856, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 3018711040, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.30.ffn_gate.weight", "offset": 3028148224, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 3047522304, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.30.ffn_down.weight", "offset": 3066896384, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 3092258816, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 3092275200, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 3092291584, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.31.attn_k.weight", "offset": 3099500544, "shape": [ 4096, 4096 ], "size": 7208960, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 3106709504, "shape": [ 4096, 4096 ], "size": 7208960, "type": 12 }, { "name": "blk.31.attn_output.weight", "offset": 3116146688, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.31.ffn_gate.weight", "offset": 3125583872, "shape": [ 4096, 11008 ], "size": 9437184, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 3144957952, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.31.ffn_down.weight", "offset": 3164332032, "shape": [ 11008, 4096 ], "size": 19374080, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 3189694464, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 3189710848, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "output_norm.weight", "offset": 3189727232, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "output.weight", "offset": 3189743616, "shape": [ 4096, 32000 ], "size": 16384, "type": 14 } ], "version": 2 }