yi

{ "metadata": { "general.architecture": "llama", "general.file_type": 3, "general.name": "Yi-1.5-6B-Chat", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 4, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 32, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 5000000, "llama.vocab_size": 64000, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (64000 values)", "tokenizer.ggml.token_type": "... (64000 values)", "tokenizer.ggml.tokens": "... (64000 values)" }, "num_params": 6061035520, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 163840000, "shape": [ 4096 ], "size": 163840000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 163856384, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.0.ffn_gate.weight", "offset": 192036864, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.0.ffn_up.weight", "offset": 220217344, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.0.ffn_norm.weight", "offset": 248397824, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 248414208, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.0.attn_output.weight", "offset": 249724928, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.0.attn_q.weight", "offset": 260210688, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.0.attn_v.weight", "offset": 270696448, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.1.attn_norm.weight", "offset": 272007168, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 272023552, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.1.ffn_gate.weight", "offset": 300204032, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.1.ffn_up.weight", "offset": 328384512, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.1.ffn_norm.weight", "offset": 356564992, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 356581376, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.1.attn_output.weight", "offset": 357892096, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.1.attn_q.weight", "offset": 368377856, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.1.attn_v.weight", "offset": 378863616, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.10.attn_norm.weight", "offset": 380174336, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 380190720, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.10.ffn_gate.weight", "offset": 408371200, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.10.ffn_up.weight", "offset": 436551680, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.10.ffn_norm.weight", "offset": 464732160, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 464748544, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.10.attn_output.weight", "offset": 466059264, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.10.attn_q.weight", "offset": 476545024, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.10.attn_v.weight", "offset": 487030784, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.11.attn_norm.weight", "offset": 488341504, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 488357888, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.11.ffn_gate.weight", "offset": 516538368, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.11.ffn_up.weight", "offset": 544718848, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.11.ffn_norm.weight", "offset": 572899328, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 572915712, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.11.attn_output.weight", "offset": 574226432, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.11.attn_q.weight", "offset": 584712192, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.11.attn_v.weight", "offset": 595197952, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.12.ffn_gate.weight", "offset": 596508672, "shape": [ 4096, 11008 ], "size": 1310720, "type": 3 }, { "name": "blk.12.ffn_up.weight", "offset": 624689152, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.12.attn_k.weight", "offset": 652869632, "shape": [ 4096, 512 ], "size": 28180480, "type": 3 }, { "name": "blk.12.attn_output.weight", "offset": 654180352, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.12.attn_q.weight", "offset": 664666112, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.12.attn_v.weight", "offset": 675151872, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.2.attn_norm.weight", "offset": 676462592, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 676478976, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.2.ffn_gate.weight", "offset": 704659456, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.2.ffn_up.weight", "offset": 732839936, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.2.ffn_norm.weight", "offset": 761020416, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 761036800, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.2.attn_output.weight", "offset": 762347520, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.2.attn_q.weight", "offset": 772833280, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.2.attn_v.weight", "offset": 783319040, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.3.attn_norm.weight", "offset": 784629760, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 784646144, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.3.ffn_gate.weight", "offset": 812826624, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.3.ffn_up.weight", "offset": 841007104, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.3.ffn_norm.weight", "offset": 869187584, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 869203968, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.3.attn_output.weight", "offset": 870514688, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.3.attn_q.weight", "offset": 881000448, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.3.attn_v.weight", "offset": 891486208, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.4.attn_norm.weight", "offset": 892796928, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 892813312, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.4.ffn_gate.weight", "offset": 920993792, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.4.ffn_up.weight", "offset": 949174272, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.4.ffn_norm.weight", "offset": 977354752, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 977371136, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.4.attn_output.weight", "offset": 978681856, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.4.attn_q.weight", "offset": 989167616, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.4.attn_v.weight", "offset": 999653376, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.5.attn_norm.weight", "offset": 1000964096, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1000980480, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.5.ffn_gate.weight", "offset": 1029160960, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.5.ffn_up.weight", "offset": 1057341440, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.5.ffn_norm.weight", "offset": 1085521920, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1085538304, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.5.attn_output.weight", "offset": 1086849024, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.5.attn_q.weight", "offset": 1097334784, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.5.attn_v.weight", "offset": 1107820544, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.6.attn_norm.weight", "offset": 1109131264, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1109147648, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.6.ffn_gate.weight", "offset": 1137328128, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.6.ffn_up.weight", "offset": 1165508608, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.6.ffn_norm.weight", "offset": 1193689088, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1193705472, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.6.attn_output.weight", "offset": 1195016192, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.6.attn_q.weight", "offset": 1205501952, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.6.attn_v.weight", "offset": 1215987712, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.7.attn_norm.weight", "offset": 1217298432, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1217314816, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.7.ffn_gate.weight", "offset": 1245495296, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.7.ffn_up.weight", "offset": 1273675776, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.7.ffn_norm.weight", "offset": 1301856256, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1301872640, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.7.attn_output.weight", "offset": 1303183360, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.7.attn_q.weight", "offset": 1313669120, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.7.attn_v.weight", "offset": 1324154880, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.8.attn_norm.weight", "offset": 1325465600, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1325481984, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.8.ffn_gate.weight", "offset": 1353662464, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.8.ffn_up.weight", "offset": 1381842944, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.8.ffn_norm.weight", "offset": 1410023424, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1410039808, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.8.attn_output.weight", "offset": 1411350528, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.8.attn_q.weight", "offset": 1421836288, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.8.attn_v.weight", "offset": 1432322048, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.9.attn_norm.weight", "offset": 1433632768, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1433649152, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.9.ffn_gate.weight", "offset": 1461829632, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.9.ffn_up.weight", "offset": 1490010112, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.9.ffn_norm.weight", "offset": 1518190592, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1518206976, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.9.attn_output.weight", "offset": 1519517696, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.9.attn_q.weight", "offset": 1530003456, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.9.attn_v.weight", "offset": 1540489216, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.12.attn_norm.weight", "offset": 1541799936, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1541816320, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.12.ffn_norm.weight", "offset": 1569996800, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 1570013184, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1570029568, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.13.ffn_gate.weight", "offset": 1598210048, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.13.ffn_up.weight", "offset": 1626390528, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.13.ffn_norm.weight", "offset": 1654571008, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1654587392, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.13.attn_output.weight", "offset": 1655898112, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.13.attn_q.weight", "offset": 1666383872, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.13.attn_v.weight", "offset": 1676869632, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.14.attn_norm.weight", "offset": 1678180352, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1678196736, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.14.ffn_gate.weight", "offset": 1706377216, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.14.ffn_up.weight", "offset": 1734557696, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.14.ffn_norm.weight", "offset": 1762738176, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1762754560, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.14.attn_output.weight", "offset": 1764065280, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.14.attn_q.weight", "offset": 1774551040, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.14.attn_v.weight", "offset": 1785036800, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.15.attn_norm.weight", "offset": 1786347520, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1786363904, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.15.ffn_gate.weight", "offset": 1814544384, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.15.ffn_up.weight", "offset": 1842724864, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.15.ffn_norm.weight", "offset": 1870905344, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1870921728, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.15.attn_output.weight", "offset": 1872232448, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.15.attn_q.weight", "offset": 1882718208, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.15.attn_v.weight", "offset": 1893203968, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.16.attn_norm.weight", "offset": 1894514688, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 1894531072, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.16.ffn_gate.weight", "offset": 1922711552, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.16.ffn_up.weight", "offset": 1950892032, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.16.ffn_norm.weight", "offset": 1979072512, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1979088896, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.16.attn_output.weight", "offset": 1980399616, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.16.attn_q.weight", "offset": 1990885376, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.16.attn_v.weight", "offset": 2001371136, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.17.attn_norm.weight", "offset": 2002681856, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2002698240, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.17.ffn_gate.weight", "offset": 2030878720, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.17.ffn_up.weight", "offset": 2059059200, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.17.ffn_norm.weight", "offset": 2087239680, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2087256064, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.17.attn_output.weight", "offset": 2088566784, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.17.attn_q.weight", "offset": 2099052544, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.17.attn_v.weight", "offset": 2109538304, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.18.attn_norm.weight", "offset": 2110849024, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2110865408, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.18.ffn_gate.weight", "offset": 2139045888, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.18.ffn_up.weight", "offset": 2167226368, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.18.ffn_norm.weight", "offset": 2195406848, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2195423232, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.18.attn_output.weight", "offset": 2196733952, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.18.attn_q.weight", "offset": 2207219712, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.18.attn_v.weight", "offset": 2217705472, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.19.attn_norm.weight", "offset": 2219016192, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2219032576, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.19.ffn_gate.weight", "offset": 2247213056, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.19.ffn_up.weight", "offset": 2275393536, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.19.ffn_norm.weight", "offset": 2303574016, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2303590400, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.19.attn_output.weight", "offset": 2304901120, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.19.attn_q.weight", "offset": 2315386880, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.19.attn_v.weight", "offset": 2325872640, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.20.attn_norm.weight", "offset": 2327183360, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2327199744, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.20.ffn_gate.weight", "offset": 2355380224, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.20.ffn_up.weight", "offset": 2383560704, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.20.ffn_norm.weight", "offset": 2411741184, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 2411757568, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.20.attn_output.weight", "offset": 2413068288, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.20.attn_q.weight", "offset": 2423554048, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.20.attn_v.weight", "offset": 2434039808, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.21.attn_norm.weight", "offset": 2435350528, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2435366912, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.21.ffn_gate.weight", "offset": 2463547392, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.21.ffn_up.weight", "offset": 2491727872, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.21.ffn_norm.weight", "offset": 2519908352, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2519924736, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.21.attn_output.weight", "offset": 2521235456, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.21.attn_q.weight", "offset": 2531721216, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.21.attn_v.weight", "offset": 2542206976, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.22.attn_norm.weight", "offset": 2543517696, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2543534080, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.22.ffn_gate.weight", "offset": 2571714560, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.22.ffn_up.weight", "offset": 2599895040, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.22.ffn_norm.weight", "offset": 2628075520, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2628091904, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.22.attn_output.weight", "offset": 2629402624, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.22.attn_q.weight", "offset": 2639888384, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.22.attn_v.weight", "offset": 2650374144, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.23.attn_norm.weight", "offset": 2651684864, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2651701248, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.23.ffn_gate.weight", "offset": 2679881728, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.23.ffn_up.weight", "offset": 2708062208, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.23.ffn_norm.weight", "offset": 2736242688, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2736259072, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.23.attn_output.weight", "offset": 2737569792, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.23.attn_q.weight", "offset": 2748055552, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.23.attn_v.weight", "offset": 2758541312, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.24.attn_norm.weight", "offset": 2759852032, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2759868416, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.24.ffn_gate.weight", "offset": 2788048896, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.24.ffn_up.weight", "offset": 2816229376, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.24.ffn_norm.weight", "offset": 2844409856, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 2844426240, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.24.attn_output.weight", "offset": 2845736960, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.24.attn_q.weight", "offset": 2856222720, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.24.attn_v.weight", "offset": 2866708480, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.25.attn_norm.weight", "offset": 2868019200, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 2868035584, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.25.ffn_gate.weight", "offset": 2896216064, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.25.ffn_up.weight", "offset": 2924396544, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.25.ffn_norm.weight", "offset": 2952577024, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 2952593408, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.25.attn_output.weight", "offset": 2953904128, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.25.attn_q.weight", "offset": 2964389888, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.25.attn_v.weight", "offset": 2974875648, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.26.attn_norm.weight", "offset": 2976186368, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 2976202752, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.26.ffn_gate.weight", "offset": 3004383232, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.26.ffn_up.weight", "offset": 3032563712, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.26.ffn_norm.weight", "offset": 3060744192, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3060760576, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.26.attn_output.weight", "offset": 3062071296, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.26.attn_q.weight", "offset": 3072557056, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.26.attn_v.weight", "offset": 3083042816, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.27.attn_k.weight", "offset": 3084353536, "shape": [ 4096, 512 ], "size": 1310720, "type": 3 }, { "name": "blk.27.attn_q.weight", "offset": 3085664256, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.27.attn_v.weight", "offset": 3096150016, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "output.weight", "offset": 3097460736, "shape": [ 4096, 64000 ], "size": 1310720, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 3312500736, "shape": [ 4096 ], "size": 215040000, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3312517120, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.27.ffn_gate.weight", "offset": 3340697600, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.27.ffn_up.weight", "offset": 3368878080, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.27.ffn_norm.weight", "offset": 3397058560, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 3397074944, "shape": [ 4096, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.28.attn_norm.weight", "offset": 3407560704, "shape": [ 4096 ], "size": 10485760, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3407577088, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.28.ffn_gate.weight", "offset": 3435757568, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.28.ffn_up.weight", "offset": 3463938048, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.28.ffn_norm.weight", "offset": 3492118528, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3492134912, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.28.attn_output.weight", "offset": 3493445632, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.28.attn_q.weight", "offset": 3503931392, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.28.attn_v.weight", "offset": 3514417152, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.29.attn_norm.weight", "offset": 3515727872, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3515744256, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.29.ffn_gate.weight", "offset": 3543924736, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.29.ffn_up.weight", "offset": 3572105216, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.29.ffn_norm.weight", "offset": 3600285696, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 3600302080, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.29.attn_output.weight", "offset": 3601612800, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.29.attn_q.weight", "offset": 3612098560, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.29.attn_v.weight", "offset": 3622584320, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.30.attn_norm.weight", "offset": 3623895040, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3623911424, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.30.ffn_gate.weight", "offset": 3652091904, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.30.ffn_up.weight", "offset": 3680272384, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.30.ffn_norm.weight", "offset": 3708452864, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 3708469248, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.30.attn_output.weight", "offset": 3709779968, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.30.attn_q.weight", "offset": 3720265728, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.30.attn_v.weight", "offset": 3730751488, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "blk.31.attn_norm.weight", "offset": 3732062208, "shape": [ 4096 ], "size": 1310720, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 3732078592, "shape": [ 11008, 4096 ], "size": 16384, "type": 3 }, { "name": "blk.31.ffn_gate.weight", "offset": 3760259072, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.31.ffn_up.weight", "offset": 3788439552, "shape": [ 4096, 11008 ], "size": 28180480, "type": 3 }, { "name": "blk.31.ffn_norm.weight", "offset": 3816620032, "shape": [ 4096 ], "size": 28180480, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 3816636416, "shape": [ 4096, 512 ], "size": 16384, "type": 3 }, { "name": "blk.31.attn_output.weight", "offset": 3817947136, "shape": [ 4096, 4096 ], "size": 1310720, "type": 3 }, { "name": "blk.31.attn_q.weight", "offset": 3828432896, "shape": [ 4096, 4096 ], "size": 10485760, "type": 3 }, { "name": "blk.31.attn_v.weight", "offset": 3838918656, "shape": [ 4096, 512 ], "size": 10485760, "type": 3 }, { "name": "output_norm.weight", "offset": 3840229376, "shape": [ 4096 ], "size": 1310720, "type": 0 } ], "version": 3 }