yi

{ "metadata": { "general.architecture": "llama", "general.file_type": 9, "general.name": "Yi-1.5-9B-Chat", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 4, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 48, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 5000000, "llama.vocab_size": 64000, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (64000 values)", "tokenizer.ggml.token_type": "... (64000 values)", "tokenizer.ggml.tokens": "... (64000 values)" }, "num_params": 8829407232, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 196608000, "shape": [ 4096 ], "size": 196608000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 196624384, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.0.ffn_gate.weight", "offset": 230440960, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 264257536, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 298074112, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 298090496, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.0.attn_output.weight", "offset": 299663360, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.0.attn_q.weight", "offset": 312246272, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.0.attn_v.weight", "offset": 324829184, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 326402048, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 326418432, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.1.ffn_gate.weight", "offset": 360235008, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.1.ffn_up.weight", "offset": 394051584, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.1.ffn_norm.weight", "offset": 427868160, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 427884544, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.1.attn_output.weight", "offset": 429457408, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.1.attn_q.weight", "offset": 442040320, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.1.attn_v.weight", "offset": 454623232, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 456196096, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 456212480, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.10.ffn_gate.weight", "offset": 490029056, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.10.ffn_up.weight", "offset": 523845632, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 557662208, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 557678592, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.10.attn_output.weight", "offset": 559251456, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.10.attn_q.weight", "offset": 571834368, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.10.attn_v.weight", "offset": 584417280, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 585990144, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 586006528, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.11.ffn_gate.weight", "offset": 619823104, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.11.ffn_up.weight", "offset": 653639680, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.11.ffn_norm.weight", "offset": 687456256, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 687472640, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.11.attn_output.weight", "offset": 689045504, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.11.attn_q.weight", "offset": 701628416, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.11.attn_v.weight", "offset": 714211328, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.12.ffn_gate.weight", "offset": 715784192, "shape": [ 4096, 11008 ], "size": 1572864, "type": 7 }, { "name": "blk.12.ffn_up.weight", "offset": 749600768, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.12.attn_k.weight", "offset": 783417344, "shape": [ 4096, 512 ], "size": 33816576, "type": 7 }, { "name": "blk.12.attn_output.weight", "offset": 784990208, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.12.attn_q.weight", "offset": 797573120, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.12.attn_v.weight", "offset": 810156032, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 811728896, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 811745280, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.2.ffn_gate.weight", "offset": 845561856, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.2.ffn_up.weight", "offset": 879378432, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.2.ffn_norm.weight", "offset": 913195008, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 913211392, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.2.attn_output.weight", "offset": 914784256, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.2.attn_q.weight", "offset": 927367168, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.2.attn_v.weight", "offset": 939950080, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 941522944, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 941539328, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.3.ffn_gate.weight", "offset": 975355904, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.3.ffn_up.weight", "offset": 1009172480, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.3.ffn_norm.weight", "offset": 1042989056, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1043005440, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.3.attn_output.weight", "offset": 1044578304, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.3.attn_q.weight", "offset": 1057161216, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.3.attn_v.weight", "offset": 1069744128, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 1071316992, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1071333376, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.4.ffn_gate.weight", "offset": 1105149952, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.4.ffn_up.weight", "offset": 1138966528, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.4.ffn_norm.weight", "offset": 1172783104, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1172799488, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.4.attn_output.weight", "offset": 1174372352, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.4.attn_q.weight", "offset": 1186955264, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.4.attn_v.weight", "offset": 1199538176, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 1201111040, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1201127424, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.5.ffn_gate.weight", "offset": 1234944000, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.5.ffn_up.weight", "offset": 1268760576, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.5.ffn_norm.weight", "offset": 1302577152, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1302593536, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.5.attn_output.weight", "offset": 1304166400, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.5.attn_q.weight", "offset": 1316749312, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.5.attn_v.weight", "offset": 1329332224, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 1330905088, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1330921472, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.6.ffn_gate.weight", "offset": 1364738048, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.6.ffn_up.weight", "offset": 1398554624, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.6.ffn_norm.weight", "offset": 1432371200, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1432387584, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 1433960448, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.6.attn_q.weight", "offset": 1446543360, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.6.attn_v.weight", "offset": 1459126272, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.7.attn_norm.weight", "offset": 1460699136, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1460715520, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.7.ffn_gate.weight", "offset": 1494532096, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 1528348672, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.7.ffn_norm.weight", "offset": 1562165248, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1562181632, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.7.attn_output.weight", "offset": 1563754496, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.7.attn_q.weight", "offset": 1576337408, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.7.attn_v.weight", "offset": 1588920320, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.8.attn_norm.weight", "offset": 1590493184, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1590509568, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.8.ffn_gate.weight", "offset": 1624326144, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.8.ffn_up.weight", "offset": 1658142720, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.8.ffn_norm.weight", "offset": 1691959296, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1691975680, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.8.attn_output.weight", "offset": 1693548544, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.8.attn_q.weight", "offset": 1706131456, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.8.attn_v.weight", "offset": 1718714368, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 1720287232, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1720303616, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.9.ffn_gate.weight", "offset": 1754120192, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.9.ffn_up.weight", "offset": 1787936768, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 1821753344, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1821769728, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.9.attn_output.weight", "offset": 1823342592, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.9.attn_q.weight", "offset": 1835925504, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.9.attn_v.weight", "offset": 1848508416, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 1850081280, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1850097664, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 1883914240, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 1883930624, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1883947008, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.13.ffn_gate.weight", "offset": 1917763584, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.13.ffn_up.weight", "offset": 1951580160, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 1985396736, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1985413120, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.13.attn_output.weight", "offset": 1986985984, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.13.attn_q.weight", "offset": 1999568896, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.13.attn_v.weight", "offset": 2012151808, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 2013724672, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2013741056, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.14.ffn_gate.weight", "offset": 2047557632, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.14.ffn_up.weight", "offset": 2081374208, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 2115190784, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2115207168, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.14.attn_output.weight", "offset": 2116780032, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.14.attn_q.weight", "offset": 2129362944, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.14.attn_v.weight", "offset": 2141945856, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 2143518720, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2143535104, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.15.ffn_gate.weight", "offset": 2177351680, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.15.ffn_up.weight", "offset": 2211168256, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.15.ffn_norm.weight", "offset": 2244984832, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2245001216, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.15.attn_output.weight", "offset": 2246574080, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.15.attn_q.weight", "offset": 2259156992, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.15.attn_v.weight", "offset": 2271739904, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 2273312768, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2273329152, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.16.ffn_gate.weight", "offset": 2307145728, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 2340962304, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 2374778880, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2374795264, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.16.attn_output.weight", "offset": 2376368128, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.16.attn_q.weight", "offset": 2388951040, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.16.attn_v.weight", "offset": 2401533952, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.17.attn_norm.weight", "offset": 2403106816, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2403123200, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.17.ffn_gate.weight", "offset": 2436939776, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.17.ffn_up.weight", "offset": 2470756352, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.17.ffn_norm.weight", "offset": 2504572928, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2504589312, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.17.attn_output.weight", "offset": 2506162176, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.17.attn_q.weight", "offset": 2518745088, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.17.attn_v.weight", "offset": 2531328000, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 2532900864, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2532917248, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.18.ffn_gate.weight", "offset": 2566733824, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.18.ffn_up.weight", "offset": 2600550400, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 2634366976, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2634383360, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.18.attn_output.weight", "offset": 2635956224, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.18.attn_q.weight", "offset": 2648539136, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.18.attn_v.weight", "offset": 2661122048, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.19.attn_norm.weight", "offset": 2662694912, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2662711296, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.19.ffn_gate.weight", "offset": 2696527872, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.19.ffn_up.weight", "offset": 2730344448, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.19.ffn_norm.weight", "offset": 2764161024, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2764177408, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.19.attn_output.weight", "offset": 2765750272, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.19.attn_q.weight", "offset": 2778333184, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.19.attn_v.weight", "offset": 2790916096, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 2792488960, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2792505344, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.20.ffn_gate.weight", "offset": 2826321920, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.20.ffn_up.weight", "offset": 2860138496, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.20.ffn_norm.weight", "offset": 2893955072, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 2893971456, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.20.attn_output.weight", "offset": 2895544320, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.20.attn_q.weight", "offset": 2908127232, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.20.attn_v.weight", "offset": 2920710144, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 2922283008, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2922299392, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.21.ffn_gate.weight", "offset": 2956115968, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.21.ffn_up.weight", "offset": 2989932544, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.21.ffn_norm.weight", "offset": 3023749120, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 3023765504, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 3025338368, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.21.attn_q.weight", "offset": 3037921280, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.21.attn_v.weight", "offset": 3050504192, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.22.attn_norm.weight", "offset": 3052077056, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3052093440, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.22.ffn_gate.weight", "offset": 3085910016, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.22.ffn_up.weight", "offset": 3119726592, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.22.ffn_norm.weight", "offset": 3153543168, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3153559552, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.22.attn_output.weight", "offset": 3155132416, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.22.attn_q.weight", "offset": 3167715328, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.22.attn_v.weight", "offset": 3180298240, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 3181871104, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3181887488, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.23.ffn_gate.weight", "offset": 3215704064, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.23.ffn_up.weight", "offset": 3249520640, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.23.ffn_norm.weight", "offset": 3283337216, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3283353600, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.23.attn_output.weight", "offset": 3284926464, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.23.attn_q.weight", "offset": 3297509376, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.23.attn_v.weight", "offset": 3310092288, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.24.attn_norm.weight", "offset": 3311665152, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3311681536, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.24.ffn_gate.weight", "offset": 3345498112, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.24.ffn_up.weight", "offset": 3379314688, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.24.ffn_norm.weight", "offset": 3413131264, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 3413147648, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.24.attn_output.weight", "offset": 3414720512, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.24.attn_q.weight", "offset": 3427303424, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.24.attn_v.weight", "offset": 3439886336, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.25.attn_norm.weight", "offset": 3441459200, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3441475584, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.25.ffn_gate.weight", "offset": 3475292160, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.25.ffn_up.weight", "offset": 3509108736, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.25.ffn_norm.weight", "offset": 3542925312, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3542941696, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.25.attn_output.weight", "offset": 3544514560, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.25.attn_q.weight", "offset": 3557097472, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.25.attn_v.weight", "offset": 3569680384, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.26.attn_norm.weight", "offset": 3571253248, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3571269632, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.26.ffn_gate.weight", "offset": 3605086208, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.26.ffn_up.weight", "offset": 3638902784, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.26.ffn_norm.weight", "offset": 3672719360, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3672735744, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.26.attn_output.weight", "offset": 3674308608, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.26.attn_q.weight", "offset": 3686891520, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.26.attn_v.weight", "offset": 3699474432, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.27.attn_k.weight", "offset": 3701047296, "shape": [ 4096, 512 ], "size": 1572864, "type": 7 }, { "name": "blk.27.attn_q.weight", "offset": 3702620160, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.27.attn_v.weight", "offset": 3715203072, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.27.attn_norm.weight", "offset": 3716775936, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3716792320, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.27.ffn_gate.weight", "offset": 3750608896, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.27.ffn_up.weight", "offset": 3784425472, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.27.ffn_norm.weight", "offset": 3818242048, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 3818258432, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.28.attn_norm.weight", "offset": 3830841344, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3830857728, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.28.ffn_gate.weight", "offset": 3864674304, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.28.ffn_up.weight", "offset": 3898490880, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.28.ffn_norm.weight", "offset": 3932307456, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3932323840, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.28.attn_output.weight", "offset": 3933896704, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.28.attn_q.weight", "offset": 3946479616, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.28.attn_v.weight", "offset": 3959062528, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.29.attn_norm.weight", "offset": 3960635392, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3960651776, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.29.ffn_gate.weight", "offset": 3994468352, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.29.ffn_up.weight", "offset": 4028284928, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.29.ffn_norm.weight", "offset": 4062101504, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 4062117888, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.29.attn_output.weight", "offset": 4063690752, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.29.attn_q.weight", "offset": 4076273664, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.29.attn_v.weight", "offset": 4088856576, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.30.attn_norm.weight", "offset": 4090429440, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 4090445824, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.30.ffn_gate.weight", "offset": 4124262400, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.30.ffn_up.weight", "offset": 4158078976, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.30.ffn_norm.weight", "offset": 4191895552, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 4191911936, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.30.attn_output.weight", "offset": 4193484800, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.30.attn_q.weight", "offset": 4206067712, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.30.attn_v.weight", "offset": 4218650624, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.31.attn_norm.weight", "offset": 4220223488, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 4220239872, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.31.ffn_gate.weight", "offset": 4254056448, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.31.ffn_up.weight", "offset": 4287873024, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.31.ffn_norm.weight", "offset": 4321689600, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 4321705984, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.31.attn_output.weight", "offset": 4323278848, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.31.attn_q.weight", "offset": 4335861760, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.31.attn_v.weight", "offset": 4348444672, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.32.attn_norm.weight", "offset": 4350017536, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 4350033920, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.32.ffn_gate.weight", "offset": 4383850496, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.32.ffn_up.weight", "offset": 4417667072, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.32.ffn_norm.weight", "offset": 4451483648, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 4451500032, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.32.attn_output.weight", "offset": 4453072896, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.32.attn_q.weight", "offset": 4465655808, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.32.attn_v.weight", "offset": 4478238720, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.33.attn_norm.weight", "offset": 4479811584, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 4479827968, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.33.ffn_gate.weight", "offset": 4513644544, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.33.ffn_up.weight", "offset": 4547461120, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.33.ffn_norm.weight", "offset": 4581277696, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 4581294080, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.33.attn_output.weight", "offset": 4582866944, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.33.attn_q.weight", "offset": 4595449856, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.33.attn_v.weight", "offset": 4608032768, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.34.attn_norm.weight", "offset": 4609605632, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 4609622016, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.34.ffn_gate.weight", "offset": 4643438592, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.34.ffn_up.weight", "offset": 4677255168, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.34.ffn_norm.weight", "offset": 4711071744, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 4711088128, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.34.attn_output.weight", "offset": 4712660992, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.34.attn_q.weight", "offset": 4725243904, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.34.attn_v.weight", "offset": 4737826816, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.35.attn_norm.weight", "offset": 4739399680, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 4739416064, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.35.ffn_gate.weight", "offset": 4773232640, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.35.ffn_up.weight", "offset": 4807049216, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.35.ffn_norm.weight", "offset": 4840865792, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 4840882176, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.35.attn_output.weight", "offset": 4842455040, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.35.attn_q.weight", "offset": 4855037952, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.35.attn_v.weight", "offset": 4867620864, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.36.attn_norm.weight", "offset": 4869193728, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 4869210112, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.36.ffn_gate.weight", "offset": 4903026688, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.36.ffn_up.weight", "offset": 4936843264, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.36.ffn_norm.weight", "offset": 4970659840, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 4970676224, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.36.attn_output.weight", "offset": 4972249088, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.36.attn_q.weight", "offset": 4984832000, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.36.attn_v.weight", "offset": 4997414912, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.37.attn_norm.weight", "offset": 4998987776, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 4999004160, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.37.ffn_gate.weight", "offset": 5032820736, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.37.ffn_up.weight", "offset": 5066637312, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.37.ffn_norm.weight", "offset": 5100453888, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 5100470272, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.37.attn_output.weight", "offset": 5102043136, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.37.attn_q.weight", "offset": 5114626048, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.37.attn_v.weight", "offset": 5127208960, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.38.attn_norm.weight", "offset": 5128781824, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 5128798208, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.38.ffn_gate.weight", "offset": 5162614784, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.38.ffn_up.weight", "offset": 5196431360, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.38.ffn_norm.weight", "offset": 5230247936, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 5230264320, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.38.attn_output.weight", "offset": 5231837184, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.38.attn_q.weight", "offset": 5244420096, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.38.attn_v.weight", "offset": 5257003008, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.39.attn_norm.weight", "offset": 5258575872, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 5258592256, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.39.ffn_gate.weight", "offset": 5292408832, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.39.ffn_up.weight", "offset": 5326225408, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.39.ffn_norm.weight", "offset": 5360041984, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 5360058368, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.39.attn_output.weight", "offset": 5361631232, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.39.attn_q.weight", "offset": 5374214144, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.39.attn_v.weight", "offset": 5386797056, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.40.attn_norm.weight", "offset": 5388369920, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 5388386304, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.40.ffn_gate.weight", "offset": 5422202880, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.40.ffn_up.weight", "offset": 5456019456, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.40.ffn_norm.weight", "offset": 5489836032, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 5489852416, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.40.attn_output.weight", "offset": 5491425280, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.40.attn_q.weight", "offset": 5504008192, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.40.attn_v.weight", "offset": 5516591104, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.41.ffn_gate.weight", "offset": 5518163968, "shape": [ 4096, 11008 ], "size": 1572864, "type": 7 }, { "name": "blk.41.attn_k.weight", "offset": 5551980544, "shape": [ 4096, 512 ], "size": 33816576, "type": 7 }, { "name": "blk.41.attn_output.weight", "offset": 5553553408, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.41.attn_q.weight", "offset": 5566136320, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.41.attn_v.weight", "offset": 5578719232, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "output.weight", "offset": 5580292096, "shape": [ 4096, 64000 ], "size": 1572864, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 5795332096, "shape": [ 4096 ], "size": 215040000, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 5795348480, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.41.ffn_up.weight", "offset": 5829165056, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.41.ffn_norm.weight", "offset": 5862981632, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 5862998016, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 5863014400, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.42.ffn_gate.weight", "offset": 5896830976, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.42.ffn_up.weight", "offset": 5930647552, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.42.ffn_norm.weight", "offset": 5964464128, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 5964480512, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.42.attn_output.weight", "offset": 5966053376, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.42.attn_q.weight", "offset": 5978636288, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.42.attn_v.weight", "offset": 5991219200, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.43.attn_norm.weight", "offset": 5992792064, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 5992808448, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.43.ffn_gate.weight", "offset": 6026625024, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.43.ffn_up.weight", "offset": 6060441600, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.43.ffn_norm.weight", "offset": 6094258176, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 6094274560, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.43.attn_output.weight", "offset": 6095847424, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.43.attn_q.weight", "offset": 6108430336, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.43.attn_v.weight", "offset": 6121013248, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.44.attn_norm.weight", "offset": 6122586112, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 6122602496, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.44.ffn_gate.weight", "offset": 6156419072, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.44.ffn_up.weight", "offset": 6190235648, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.44.ffn_norm.weight", "offset": 6224052224, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 6224068608, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.44.attn_output.weight", "offset": 6225641472, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.44.attn_q.weight", "offset": 6238224384, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.44.attn_v.weight", "offset": 6250807296, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.45.attn_norm.weight", "offset": 6252380160, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 6252396544, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.45.ffn_gate.weight", "offset": 6286213120, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.45.ffn_up.weight", "offset": 6320029696, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.45.ffn_norm.weight", "offset": 6353846272, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 6353862656, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.45.attn_output.weight", "offset": 6355435520, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.45.attn_q.weight", "offset": 6368018432, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.45.attn_v.weight", "offset": 6380601344, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.46.attn_norm.weight", "offset": 6382174208, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 6382190592, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.46.ffn_gate.weight", "offset": 6416007168, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.46.ffn_up.weight", "offset": 6449823744, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.46.ffn_norm.weight", "offset": 6483640320, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 6483656704, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.46.attn_output.weight", "offset": 6485229568, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.46.attn_q.weight", "offset": 6497812480, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.46.attn_v.weight", "offset": 6510395392, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "blk.47.attn_norm.weight", "offset": 6511968256, "shape": [ 4096 ], "size": 1572864, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 6511984640, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.47.ffn_gate.weight", "offset": 6545801216, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.47.ffn_up.weight", "offset": 6579617792, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.47.ffn_norm.weight", "offset": 6613434368, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 6613450752, "shape": [ 4096, 512 ], "size": 16384, "type": 7 }, { "name": "blk.47.attn_output.weight", "offset": 6615023616, "shape": [ 4096, 4096 ], "size": 1572864, "type": 7 }, { "name": "blk.47.attn_q.weight", "offset": 6627606528, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.47.attn_v.weight", "offset": 6640189440, "shape": [ 4096, 512 ], "size": 12582912, "type": 7 }, { "name": "output_norm.weight", "offset": 6641762304, "shape": [ 4096 ], "size": 1572864, "type": 0 } ], "version": 3 }