mixtral

{ "metadata": { "general.architecture": "llama", "general.file_type": 10, "general.name": "mistralai", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 32, "llama.context_length": 32768, "llama.embedding_length": 4096, "llama.expert_count": 8, "llama.expert_used_count": 2, "llama.feed_forward_length": 14336, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.merges": "... (58980 values)", "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32000 values)", "tokenizer.ggml.token_type": "... (32000 values)", "tokenizer.ggml.tokens": "... (32000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 46702792704, "tensors": [ { "name": "blk.0.ffn_gate.0.weight", "offset": 43008000, "shape": [ 4096, 14336 ], "size": 43008000, "type": 10 }, { "name": "blk.0.ffn_down.0.weight", "offset": 62275584, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_up.0.weight", "offset": 81543168, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_gate.1.weight", "offset": 100810752, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_down.1.weight", "offset": 120078336, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_up.1.weight", "offset": 139345920, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_gate.2.weight", "offset": 158613504, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_down.2.weight", "offset": 177881088, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_up.2.weight", "offset": 197148672, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_gate.3.weight", "offset": 216416256, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_down.3.weight", "offset": 235683840, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_up.3.weight", "offset": 254951424, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_gate.4.weight", "offset": 274219008, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_down.4.weight", "offset": 293486592, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_up.4.weight", "offset": 312754176, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_gate.5.weight", "offset": 332021760, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_down.5.weight", "offset": 351289344, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_up.5.weight", "offset": 370556928, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_gate.6.weight", "offset": 389824512, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_down.6.weight", "offset": 409092096, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_up.6.weight", "offset": 428359680, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_gate.7.weight", "offset": 447627264, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_down.7.weight", "offset": 466894848, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_up.7.weight", "offset": 486162432, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.0.ffn_gate_inp.weight", "offset": 505430016, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.0.attn_norm.weight", "offset": 505495552, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 505511936, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 505528320, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 509984768, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.0.attn_q.weight", "offset": 517193728, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.0.attn_v.weight", "offset": 522698752, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.1.ffn_gate.0.weight", "offset": 527155200, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.1.ffn_down.0.weight", "offset": 546422784, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_up.0.weight", "offset": 565690368, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_gate.1.weight", "offset": 584957952, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_down.1.weight", "offset": 604225536, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_up.1.weight", "offset": 623493120, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_gate.2.weight", "offset": 642760704, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_down.2.weight", "offset": 662028288, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_up.2.weight", "offset": 681295872, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_gate.3.weight", "offset": 700563456, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_down.3.weight", "offset": 719831040, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_up.3.weight", "offset": 739098624, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_gate.4.weight", "offset": 758366208, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_down.4.weight", "offset": 777633792, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 796901376, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.1.attn_k.weight", "offset": 796966912, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 801423360, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.1.attn_q.weight", "offset": 808632320, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.1.attn_v.weight", "offset": 814137344, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.1.ffn_up.4.weight", "offset": 818593792, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.1.ffn_gate.5.weight", "offset": 837861376, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_down.5.weight", "offset": 857128960, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_up.5.weight", "offset": 876396544, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_gate.6.weight", "offset": 895664128, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_down.6.weight", "offset": 914931712, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_up.6.weight", "offset": 934199296, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_gate.7.weight", "offset": 953466880, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_down.7.weight", "offset": 972734464, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.1.ffn_up.7.weight", "offset": 992002048, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.1.attn_norm.weight", "offset": 1011269632, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 1011286016, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.ffn_gate.0.weight", "offset": 1011302400, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.2.ffn_down.0.weight", "offset": 1030569984, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_up.0.weight", "offset": 1049837568, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_gate.1.weight", "offset": 1069105152, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_down.1.weight", "offset": 1088372736, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_up.1.weight", "offset": 1107640320, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_gate.2.weight", "offset": 1126907904, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_down.2.weight", "offset": 1146175488, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_up.2.weight", "offset": 1165443072, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_gate.3.weight", "offset": 1184710656, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_down.3.weight", "offset": 1203978240, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_up.3.weight", "offset": 1223245824, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_gate.4.weight", "offset": 1242513408, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_down.4.weight", "offset": 1261780992, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_up.4.weight", "offset": 1281048576, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "\u0000\u0000blk.6.ffn_down.6.weig", "offset": 1300316160, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_down.5.weight", "offset": 1319583744, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_up.5.weight", "offset": 1338851328, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_gate.6.weight", "offset": 1358118912, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_down.6.weight", "offset": 1377386496, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_up.6.weight", "offset": 1396654080, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_gate.7.weight", "offset": 1415921664, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_down.7.weight", "offset": 1435189248, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_up.7.weight", "offset": 1454456832, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 1473724416, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 1473789952, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 1473806336, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1473822720, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 1478279168, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.2.attn_q.weight", "offset": 1485488128, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.2.attn_v.weight", "offset": 1490993152, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.3.ffn_gate.0.weight", "offset": 1495449600, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.3.ffn_down.0.weight", "offset": 1514717184, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_up.0.weight", "offset": 1533984768, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_gate.1.weight", "offset": 1553252352, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_down.1.weight", "offset": 1572519936, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_up.1.weight", "offset": 1591787520, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_gate.2.weight", "offset": 1611055104, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 1630322688, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.3.attn_k.weight", "offset": 1630388224, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 1634844672, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.3.attn_q.weight", "offset": 1642053632, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.3.attn_v.weight", "offset": 1647558656, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.3.ffn_down.2.weight", "offset": 1652015104, "shape": [ 14336, 4096 ], "size": 4456448, "type": 10 }, { "name": "blk.3.ffn_up.2.weight", "offset": 1671282688, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_gate.3.weight", "offset": 1690550272, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_down.3.weight", "offset": 1709817856, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_up.3.weight", "offset": 1729085440, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_gate.4.weight", "offset": 1748353024, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_down.4.weight", "offset": 1767620608, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_up.4.weight", "offset": 1786888192, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_gate.5.weight", "offset": 1806155776, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_down.5.weight", "offset": 1825423360, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_up.5.weight", "offset": 1844690944, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_gate.6.weight", "offset": 1863958528, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_down.6.weight", "offset": 1883226112, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_up.6.weight", "offset": 1902493696, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_gate.7.weight", "offset": 1921761280, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_down.7.weight", "offset": 1941028864, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.3.ffn_up.7.weight", "offset": 1960296448, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.3.attn_norm.weight", "offset": 1979564032, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 1979580416, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.ffn_gate.0.weight", "offset": 1979596800, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.4.ffn_down.0.weight", "offset": 1998864384, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_up.0.weight", "offset": 2018131968, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_gate.1.weight", "offset": 2037399552, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_down.1.weight", "offset": 2056667136, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_up.1.weight", "offset": 2075934720, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_gate.2.weight", "offset": 2095202304, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_down.2.weight", "offset": 2114469888, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_up.2.weight", "offset": 2133737472, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_gate.3.weight", "offset": 2153005056, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_down.3.weight", "offset": 2172272640, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_up.3.weight", "offset": 2191540224, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_gate.4.weight", "offset": 2210807808, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_down.4.weight", "offset": 2230075392, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_up.4.weight", "offset": 2249342976, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_gate.5.weight", "offset": 2268610560, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_down.5.weight", "offset": 2287878144, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_up.5.weight", "offset": 2307145728, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_gate.6.weight", "offset": 2326413312, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_down.6.weight", "offset": 2345680896, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_up.6.weight", "offset": 2364948480, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_gate.7.weight", "offset": 2384216064, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_down.7.weight", "offset": 2403483648, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_up.7.weight", "offset": 2422751232, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 2442018816, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 2442084352, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 2442100736, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 2442117120, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 2446573568, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.4.attn_q.weight", "offset": 2453782528, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.4.attn_v.weight", "offset": 2459287552, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 2463744000, "shape": [ 4096, 8 ], "size": 4456448, "type": 1 }, { "name": "blk.5.attn_k.weight", "offset": 2463809536, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 2468265984, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.5.attn_q.weight", "offset": 2475474944, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.5.attn_v.weight", "offset": 2480979968, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.5.ffn_gate.0.weight", "offset": 2485436416, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.5.ffn_down.0.weight", "offset": 2504704000, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_up.0.weight", "offset": 2523971584, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_gate.1.weight", "offset": 2543239168, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_down.1.weight", "offset": 2562506752, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_up.1.weight", "offset": 2581774336, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_gate.2.weight", "offset": 2601041920, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_down.2.weight", "offset": 2620309504, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_up.2.weight", "offset": 2639577088, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_gate.3.weight", "offset": 2658844672, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_down.3.weight", "offset": 2678112256, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_up.3.weight", "offset": 2697379840, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_gate.4.weight", "offset": 2716647424, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_down.4.weight", "offset": 2735915008, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_up.4.weight", "offset": 2755182592, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_gate.5.weight", "offset": 2774450176, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_down.5.weight", "offset": 2793717760, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_up.5.weight", "offset": 2812985344, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_gate.6.weight", "offset": 2832252928, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_down.6.weight", "offset": 2851520512, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_up.6.weight", "offset": 2870788096, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_gate.7.weight", "offset": 2890055680, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_down.7.weight", "offset": 2909323264, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.5.ffn_up.7.weight", "offset": 2928590848, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.5.attn_norm.weight", "offset": 2947858432, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 2947874816, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.6.ffn_gate.0.weight", "offset": 2947891200, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.6.ffn_down.0.weight", "offset": 2967158784, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_up.0.weight", "offset": 2986426368, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_gate.1.weight", "offset": 3005693952, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_down.1.weight", "offset": 3024961536, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_up.1.weight", "offset": 3044229120, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_gate.2.weight", "offset": 3063496704, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_down.2.weight", "offset": 3082764288, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_up.2.weight", "offset": 3102031872, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_gate.3.weight", "offset": 3121299456, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_down.3.weight", "offset": 3140567040, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_up.3.weight", "offset": 3159834624, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_gate.4.weight", "offset": 3179102208, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_down.4.weight", "offset": 3198369792, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_up.4.weight", "offset": 3217637376, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_gate.5.weight", "offset": 3236904960, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_down.5.weight", "offset": 3256172544, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 3275440128, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.6.attn_k.weight", "offset": 3275505664, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 3279962112, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.6.attn_q.weight", "offset": 3287171072, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.6.attn_v.weight", "offset": 3292676096, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.6.ffn_up.5.weight", "offset": 3297132544, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.6.ffn_gate.6.weight", "offset": 3316400128, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "\u0000\n\u0000\u0000\u0000\u0000\ufffd$A\u0001\u0000\u0000\u0000\u0016\u0000\u0000\u0000\u0000\u0000\u0000\u0000bl", "offset": 3335667712, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_up.6.weight", "offset": 3354935296, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_gate.7.weight", "offset": 3374202880, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_down.7.weight", "offset": 3393470464, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.6.ffn_up.7.weight", "offset": 3412738048, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.6.attn_norm.weight", "offset": 3432005632, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 3432022016, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.ffn_gate.0.weight", "offset": 3432038400, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.7.ffn_down.0.weight", "offset": 3451305984, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_up.0.weight", "offset": 3470573568, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_gate.1.weight", "offset": 3489841152, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_down.1.weight", "offset": 3509108736, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_up.1.weight", "offset": 3528376320, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_gate.2.weight", "offset": 3547643904, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_down.2.weight", "offset": 3566911488, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_up.2.weight", "offset": 3586179072, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_gate.3.weight", "offset": 3605446656, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_down.3.weight", "offset": 3624714240, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_up.3.weight", "offset": 3643981824, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_gate.4.weight", "offset": 3663249408, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_down.4.weight", "offset": 3682516992, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_up.4.weight", "offset": 3701784576, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_gate.5.weight", "offset": 3721052160, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_down.5.weight", "offset": 3740319744, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_up.5.weight", "offset": 3759587328, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_gate.6.weight", "offset": 3778854912, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_down.6.weight", "offset": 3798122496, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_up.6.weight", "offset": 3817390080, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_gate.7.weight", "offset": 3836657664, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_down.7.weight", "offset": 3855925248, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_up.7.weight", "offset": 3875192832, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 3894460416, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.7.attn_norm.weight", "offset": 3894525952, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 3894542336, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 3894558720, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 3899015168, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.7.attn_q.weight", "offset": 3906224128, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.7.attn_v.weight", "offset": 3911729152, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.8.ffn_gate.0.weight", "offset": 3916185600, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.8.ffn_down.0.weight", "offset": 3935453184, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_up.0.weight", "offset": 3954720768, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_gate.1.weight", "offset": 3973988352, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_down.1.weight", "offset": 3993255936, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_up.1.weight", "offset": 4012523520, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_gate.2.weight", "offset": 4031791104, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_down.2.weight", "offset": 4051058688, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_up.2.weight", "offset": 4070326272, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_gate.3.weight", "offset": 4089593856, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 4108861440, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.8.attn_k.weight", "offset": 4108926976, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 4113383424, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.8.attn_q.weight", "offset": 4120592384, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.8.attn_v.weight", "offset": 4126097408, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.10.ffn_gate.0.weight", "offset": 4130553856, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.10.ffn_down.0.weight", "offset": 4149821440, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_up.0.weight", "offset": 4169089024, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 4188356608, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.10.attn_k.weight", "offset": 4188422144, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 4192878592, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.10.attn_q.weight", "offset": 4200087552, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.10.attn_v.weight", "offset": 4205592576, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.8.ffn_down.3.weight", "offset": 4210049024, "shape": [ 14336, 4096 ], "size": 4456448, "type": 10 }, { "name": "blk.8.ffn_up.3.weight", "offset": 4229316608, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_gate.4.weight", "offset": 4248584192, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_down.4.weight", "offset": 4267851776, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_up.4.weight", "offset": 4287119360, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_gate.5.weight", "offset": 4306386944, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_down.5.weight", "offset": 4325654528, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_up.5.weight", "offset": 4344922112, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_gate.6.weight", "offset": 4364189696, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_down.6.weight", "offset": 4383457280, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_up.6.weight", "offset": 4402724864, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_gate.7.weight", "offset": 4421992448, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_down.7.weight", "offset": 4441260032, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.8.ffn_up.7.weight", "offset": 4460527616, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.8.attn_norm.weight", "offset": 4479795200, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 4479811584, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.ffn_gate.0.weight", "offset": 4479827968, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.9.ffn_down.0.weight", "offset": 4499095552, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_up.0.weight", "offset": 4518363136, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_gate.1.weight", "offset": 4537630720, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_down.1.weight", "offset": 4556898304, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_up.1.weight", "offset": 4576165888, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_gate.2.weight", "offset": 4595433472, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_down.2.weight", "offset": 4614701056, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_up.2.weight", "offset": 4633968640, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_gate.3.weight", "offset": 4653236224, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_down.3.weight", "offset": 4672503808, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_up.3.weight", "offset": 4691771392, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_gate.4.weight", "offset": 4711038976, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_down.4.weight", "offset": 4730306560, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_up.4.weight", "offset": 4749574144, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_gate.5.weight", "offset": 4768841728, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_down.5.weight", "offset": 4788109312, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_up.5.weight", "offset": 4807376896, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_gate.6.weight", "offset": 4826644480, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_down.6.weight", "offset": 4845912064, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_up.6.weight", "offset": 4865179648, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_gate.7.weight", "offset": 4884447232, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_down.7.weight", "offset": 4903714816, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_up.7.weight", "offset": 4922982400, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 4942249984, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 4942315520, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 4942331904, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 4942348288, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 4946804736, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.9.attn_q.weight", "offset": 4954013696, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.9.attn_v.weight", "offset": 4959518720, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.10.ffn_gate.1.weight", "offset": 4963975168, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.10.ffn_down.1.weight", "offset": 4983242752, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_up.1.weight", "offset": 5002510336, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_gate.2.weight", "offset": 5021777920, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_down.2.weight", "offset": 5041045504, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_up.2.weight", "offset": 5060313088, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_gate.3.weight", "offset": 5079580672, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_down.3.weight", "offset": 5098848256, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_up.3.weight", "offset": 5118115840, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_gate.4.weight", "offset": 5137383424, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_down.4.weight", "offset": 5156651008, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_up.4.weight", "offset": 5175918592, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_gate.5.weight", "offset": 5195186176, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_down.5.weight", "offset": 5214453760, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_up.5.weight", "offset": 5233721344, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_gate.6.weight", "offset": 5252988928, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_down.6.weight", "offset": 5272256512, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_up.6.weight", "offset": 5291524096, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_gate.7.weight", "offset": 5310791680, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_down.7.weight", "offset": 5330059264, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.10.ffn_up.7.weight", "offset": 5349326848, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.10.attn_norm.weight", "offset": 5368594432, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 5368610816, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.11.ffn_gate.0.weight", "offset": 5368627200, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.11.ffn_down.0.weight", "offset": 5387894784, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_up.0.weight", "offset": 5407162368, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_gate.1.weight", "offset": 5426429952, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_down.1.weight", "offset": 5445697536, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_up.1.weight", "offset": 5464965120, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_gate.2.weight", "offset": 5484232704, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_down.2.weight", "offset": 5503500288, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_up.2.weight", "offset": 5522767872, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_gate.3.weight", "offset": 5542035456, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_down.3.weight", "offset": 5561303040, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_up.3.weight", "offset": 5580570624, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_gate.4.weight", "offset": 5599838208, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_down.4.weight", "offset": 5619105792, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_up.4.weight", "offset": 5638373376, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_gate.5.weight", "offset": 5657640960, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_down.5.weight", "offset": 5676908544, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_up.5.weight", "offset": 5696176128, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_gate.6.weight", "offset": 5715443712, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_down.6.weight", "offset": 5734711296, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 5753978880, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.11.attn_k.weight", "offset": 5754044416, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 5758500864, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.11.attn_q.weight", "offset": 5765709824, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.11.attn_v.weight", "offset": 5771214848, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.11.ffn_up.6.weight", "offset": 5775671296, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.11.ffn_gate.7.weight", "offset": 5794938880, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_down.7.weight", "offset": 5814206464, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.11.ffn_up.7.weight", "offset": 5833474048, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.11.attn_norm.weight", "offset": 5852741632, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 5852758016, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.ffn_gate.0.weight", "offset": 5852774400, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.12.ffn_down.0.weight", "offset": 5872041984, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_up.0.weight", "offset": 5891309568, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_gate.1.weight", "offset": 5910577152, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_down.1.weight", "offset": 5929844736, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_up.1.weight", "offset": 5949112320, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_gate.2.weight", "offset": 5968379904, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_down.2.weight", "offset": 5987647488, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_up.2.weight", "offset": 6006915072, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_gate.3.weight", "offset": 6026182656, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_down.3.weight", "offset": 6045450240, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_up.3.weight", "offset": 6064717824, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_gate.4.weight", "offset": 6083985408, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_down.4.weight", "offset": 6103252992, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_up.4.weight", "offset": 6122520576, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_gate.5.weight", "offset": 6141788160, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_down.5.weight", "offset": 6161055744, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_up.5.weight", "offset": 6180323328, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_gate.6.weight", "offset": 6199590912, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_down.6.weight", "offset": 6218858496, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_up.6.weight", "offset": 6238126080, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_gate.7.weight", "offset": 6257393664, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_down.7.weight", "offset": 6276661248, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_up.7.weight", "offset": 6295928832, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 6315196416, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 6315261952, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 6315278336, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 6315294720, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 6319751168, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.12.attn_q.weight", "offset": 6326960128, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.12.attn_v.weight", "offset": 6332465152, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.13.ffn_gate.0.weight", "offset": 6336921600, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.13.ffn_down.0.weight", "offset": 6356189184, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_up.0.weight", "offset": 6375456768, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_gate.1.weight", "offset": 6394724352, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_down.1.weight", "offset": 6413991936, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_up.1.weight", "offset": 6433259520, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_gate.2.weight", "offset": 6452527104, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_down.2.weight", "offset": 6471794688, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_up.2.weight", "offset": 6491062272, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_gate.3.weight", "offset": 6510329856, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_down.3.weight", "offset": 6529597440, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_up.3.weight", "offset": 6548865024, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_gate.4.weight", "offset": 6568132608, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 6587400192, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.13.attn_k.weight", "offset": 6587465728, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 6591922176, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.13.attn_q.weight", "offset": 6599131136, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.13.attn_v.weight", "offset": 6604636160, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.13.ffn_down.4.weight", "offset": 6609092608, "shape": [ 14336, 4096 ], "size": 4456448, "type": 10 }, { "name": "blk.13.ffn_up.4.weight", "offset": 6628360192, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_gate.5.weight", "offset": 6647627776, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_down.5.weight", "offset": 6666895360, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_up.5.weight", "offset": 6686162944, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_gate.6.weight", "offset": 6705430528, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_down.6.weight", "offset": 6724698112, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_up.6.weight", "offset": 6743965696, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_gate.7.weight", "offset": 6763233280, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_down.7.weight", "offset": 6782500864, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.13.ffn_up.7.weight", "offset": 6801768448, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.13.attn_norm.weight", "offset": 6821036032, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 6821052416, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.ffn_gate.0.weight", "offset": 6821068800, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.14.ffn_down.0.weight", "offset": 6840336384, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_up.0.weight", "offset": 6859603968, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_gate.1.weight", "offset": 6878871552, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_down.1.weight", "offset": 6898139136, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_up.1.weight", "offset": 6917406720, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_gate.2.weight", "offset": 6936674304, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_down.2.weight", "offset": 6955941888, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_up.2.weight", "offset": 6975209472, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_gate.3.weight", "offset": 6994477056, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_down.3.weight", "offset": 7013744640, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_up.3.weight", "offset": 7033012224, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_gate.4.weight", "offset": 7052279808, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_down.4.weight", "offset": 7071547392, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_up.4.weight", "offset": 7090814976, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_gate.5.weight", "offset": 7110082560, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_down.5.weight", "offset": 7129350144, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_up.5.weight", "offset": 7148617728, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_gate.6.weight", "offset": 7167885312, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_down.6.weight", "offset": 7187152896, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_up.6.weight", "offset": 7206420480, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_gate.7.weight", "offset": 7225688064, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_down.7.weight", "offset": 7244955648, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_up.7.weight", "offset": 7264223232, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 7283490816, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 7283556352, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 7283572736, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 7283589120, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 7288045568, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.14.attn_q.weight", "offset": 7295254528, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.14.attn_v.weight", "offset": 7300759552, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.15.ffn_gate.0.weight", "offset": 7305216000, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.15.ffn_down.0.weight", "offset": 7324483584, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_up.0.weight", "offset": 7343751168, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_gate.1.weight", "offset": 7363018752, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_down.1.weight", "offset": 7382286336, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_up.1.weight", "offset": 7401553920, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 7420821504, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.15.attn_k.weight", "offset": 7420887040, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "\u0000\u0000\ufffd\ufffd6\u0002\u0000\u0000\u0000\u0018\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.19.f", "offset": 7425343488, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.15.attn_q.weight", "offset": 7432552448, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.15.attn_v.weight", "offset": 7438057472, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.15.ffn_gate.2.weight", "offset": 7442513920, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.15.ffn_down.2.weight", "offset": 7461781504, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_up.2.weight", "offset": 7481049088, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_gate.3.weight", "offset": 7500316672, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_down.3.weight", "offset": 7519584256, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_up.3.weight", "offset": 7538851840, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_gate.4.weight", "offset": 7558119424, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_down.4.weight", "offset": 7577387008, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_up.4.weight", "offset": 7596654592, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_gate.5.weight", "offset": 7615922176, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_down.5.weight", "offset": 7635189760, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_up.5.weight", "offset": 7654457344, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_gate.6.weight", "offset": 7673724928, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_down.6.weight", "offset": 7692992512, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_up.6.weight", "offset": 7712260096, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_gate.7.weight", "offset": 7731527680, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_down.7.weight", "offset": 7750795264, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.15.ffn_up.7.weight", "offset": 7770062848, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.15.attn_norm.weight", "offset": 7789330432, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 7789346816, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.16.ffn_gate.0.weight", "offset": 7789363200, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.16.ffn_down.0.weight", "offset": 7808630784, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_up.0.weight", "offset": 7827898368, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_gate.1.weight", "offset": 7847165952, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_down.1.weight", "offset": 7866433536, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_up.1.weight", "offset": 7885701120, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_gate.2.weight", "offset": 7904968704, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_down.2.weight", "offset": 7924236288, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_up.2.weight", "offset": 7943503872, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_gate.3.weight", "offset": 7962771456, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_down.3.weight", "offset": 7982039040, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_up.3.weight", "offset": 8001306624, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_gate.4.weight", "offset": 8020574208, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_down.4.weight", "offset": 8039841792, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_up.4.weight", "offset": 8059109376, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_gate.5.weight", "offset": 8078376960, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_down.5.weight", "offset": 8097644544, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_up.5.weight", "offset": 8116912128, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_gate.6.weight", "offset": 8136179712, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_down.6.weight", "offset": 8155447296, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_up.6.weight", "offset": 8174714880, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_gate.7.weight", "offset": 8193982464, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_down.7.weight", "offset": 8213250048, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 8232517632, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.16.attn_k.weight", "offset": 8232583168, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 8237039616, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.16.attn_q.weight", "offset": 8244248576, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.16.attn_v.weight", "offset": 8249753600, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.16.ffn_up.7.weight", "offset": 8254210048, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.16.attn_norm.weight", "offset": 8273477632, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 8273494016, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.ffn_gate.0.weight", "offset": 8273510400, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.17.ffn_down.0.weight", "offset": 8292777984, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_up.0.weight", "offset": 8312045568, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_gate.1.weight", "offset": 8331313152, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_down.1.weight", "offset": 8350580736, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_up.1.weight", "offset": 8369848320, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_gate.2.weight", "offset": 8389115904, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_down.2.weight", "offset": 8408383488, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_up.2.weight", "offset": 8427651072, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_gate.3.weight", "offset": 8446918656, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_down.3.weight", "offset": 8466186240, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_up.3.weight", "offset": 8485453824, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_gate.4.weight", "offset": 8504721408, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_down.4.weight", "offset": 8523988992, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_up.4.weight", "offset": 8543256576, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_gate.5.weight", "offset": 8562524160, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_down.5.weight", "offset": 8581791744, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_up.5.weight", "offset": 8601059328, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_gate.6.weight", "offset": 8620326912, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_down.6.weight", "offset": 8639594496, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_up.6.weight", "offset": 8658862080, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_gate.7.weight", "offset": 8678129664, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_down.7.weight", "offset": 8697397248, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_up.7.weight", "offset": 8716664832, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 8735932416, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 8735997952, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 8736014336, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 8736030720, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 8740487168, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.17.attn_q.weight", "offset": 8747696128, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.17.attn_v.weight", "offset": 8753201152, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.18.ffn_gate.0.weight", "offset": 8757657600, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.18.ffn_down.0.weight", "offset": 8776925184, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_up.0.weight", "offset": 8796192768, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_gate.1.weight", "offset": 8815460352, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_down.1.weight", "offset": 8834727936, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_up.1.weight", "offset": 8853995520, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_gate.2.weight", "offset": 8873263104, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_down.2.weight", "offset": 8892530688, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_up.2.weight", "offset": 8911798272, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_gate.3.weight", "offset": 8931065856, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_down.3.weight", "offset": 8950333440, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_up.3.weight", "offset": 8969601024, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_gate.4.weight", "offset": 8988868608, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_down.4.weight", "offset": 9008136192, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_up.4.weight", "offset": 9027403776, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_gate.5.weight", "offset": 9046671360, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 9065938944, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.18.attn_k.weight", "offset": 9066004480, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 9070460928, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.18.attn_q.weight", "offset": 9077669888, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.18.attn_v.weight", "offset": 9083174912, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.18.ffn_down.5.weight", "offset": 9087631360, "shape": [ 14336, 4096 ], "size": 4456448, "type": 10 }, { "name": "blk.18.ffn_up.5.weight", "offset": 9106898944, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_gate.6.weight", "offset": 9126166528, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_down.6.weight", "offset": 9145434112, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_up.6.weight", "offset": 9164701696, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_gate.7.weight", "offset": 9183969280, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_down.7.weight", "offset": 9203236864, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.18.ffn_up.7.weight", "offset": 9222504448, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.18.attn_norm.weight", "offset": 9241772032, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 9241788416, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.ffn_gate.0.weight", "offset": 9241804800, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.19.ffn_down.0.weight", "offset": 9261072384, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_up.0.weight", "offset": 9280339968, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_gate.1.weight", "offset": 9299607552, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_down.1.weight", "offset": 9318875136, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_up.1.weight", "offset": 9338142720, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_gate.2.weight", "offset": 9357410304, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_down.2.weight", "offset": 9376677888, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_up.2.weight", "offset": 9395945472, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_gate.3.weight", "offset": 9415213056, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_down.3.weight", "offset": 9434480640, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_up.3.weight", "offset": 9453748224, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_gate.4.weight", "offset": 9473015808, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_down.4.weight", "offset": 9492283392, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_up.4.weight", "offset": 9511550976, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_gate.5.weight", "offset": 9530818560, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_down.5.weight", "offset": 9550086144, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_up.5.weight", "offset": 9569353728, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_gate.6.weight", "offset": 9588621312, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_down.6.weight", "offset": 9607888896, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_up.6.weight", "offset": 9627156480, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_gate.7.weight", "offset": 9646424064, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_down.7.weight", "offset": 9665691648, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_up.7.weight", "offset": 9684959232, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 9704226816, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.19.attn_norm.weight", "offset": 9704292352, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 9704308736, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 9704325120, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 9708781568, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.19.attn_q.weight", "offset": 9715990528, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.19.attn_v.weight", "offset": 9721495552, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.20.ffn_gate.0.weight", "offset": 9725952000, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.20.ffn_down.0.weight", "offset": 9745219584, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_up.0.weight", "offset": 9764487168, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_gate.1.weight", "offset": 9783754752, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_down.1.weight", "offset": 9803022336, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_up.1.weight", "offset": 9822289920, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_gate.2.weight", "offset": 9841557504, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_down.2.weight", "offset": 9860825088, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_up.2.weight", "offset": 9880092672, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 9899360256, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.20.attn_k.weight", "offset": 9899425792, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 9903882240, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.20.attn_q.weight", "offset": 9911091200, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.20.attn_v.weight", "offset": 9916596224, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.20.ffn_gate.3.weight", "offset": 9921052672, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.20.ffn_down.3.weight", "offset": 9940320256, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_up.3.weight", "offset": 9959587840, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_gate.4.weight", "offset": 9978855424, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_down.4.weight", "offset": 9998123008, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_up.4.weight", "offset": 10017390592, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_gate.5.weight", "offset": 10036658176, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_down.5.weight", "offset": 10055925760, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_up.5.weight", "offset": 10075193344, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_gate.6.weight", "offset": 10094460928, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_down.6.weight", "offset": 10113728512, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_up.6.weight", "offset": 10132996096, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_gate.7.weight", "offset": 10152263680, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_down.7.weight", "offset": 10171531264, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.20.ffn_up.7.weight", "offset": 10190798848, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.20.attn_norm.weight", "offset": 10210066432, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 10210082816, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.ffn_gate.0.weight", "offset": 10210099200, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.21.ffn_down.0.weight", "offset": 10229366784, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_up.0.weight", "offset": 10248634368, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_gate.1.weight", "offset": 10267901952, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_down.1.weight", "offset": 10287169536, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_up.1.weight", "offset": 10306437120, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_gate.2.weight", "offset": 10325704704, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_down.2.weight", "offset": 10344972288, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_up.2.weight", "offset": 10364239872, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_gate.3.weight", "offset": 10383507456, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_down.3.weight", "offset": 10402775040, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_up.3.weight", "offset": 10422042624, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_gate.4.weight", "offset": 10441310208, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_down.4.weight", "offset": 10460577792, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_up.4.weight", "offset": 10479845376, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_gate.5.weight", "offset": 10499112960, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_down.5.weight", "offset": 10518380544, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_up.5.weight", "offset": 10537648128, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_gate.6.weight", "offset": 10556915712, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_down.6.weight", "offset": 10576183296, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_up.6.weight", "offset": 10595450880, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_gate.7.weight", "offset": 10614718464, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_down.7.weight", "offset": 10633986048, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_up.7.weight", "offset": 10653253632, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 10672521216, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 10672586752, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 10672603136, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 10672619520, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 10677075968, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.21.attn_q.weight", "offset": 10684284928, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.21.attn_v.weight", "offset": 10689789952, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.22.ffn_gate.0.weight", "offset": 10694246400, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.22.ffn_down.0.weight", "offset": 10713513984, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 10732781568, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.22.attn_k.weight", "offset": 10732847104, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 10737303552, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.22.attn_q.weight", "offset": 10744512512, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.22.attn_v.weight", "offset": 10750017536, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.22.ffn_up.0.weight", "offset": 10754473984, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.22.ffn_gate.1.weight", "offset": 10773741568, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_down.1.weight", "offset": 10793009152, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_up.1.weight", "offset": 10812276736, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_gate.2.weight", "offset": 10831544320, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_down.2.weight", "offset": 10850811904, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_up.2.weight", "offset": 10870079488, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_gate.3.weight", "offset": 10889347072, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_down.3.weight", "offset": 10908614656, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_up.3.weight", "offset": 10927882240, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_gate.4.weight", "offset": 10947149824, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_down.4.weight", "offset": 10966417408, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_up.4.weight", "offset": 10985684992, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_gate.5.weight", "offset": 11004952576, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_down.5.weight", "offset": 11024220160, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_up.5.weight", "offset": 11043487744, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_gate.6.weight", "offset": 11062755328, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_down.6.weight", "offset": 11082022912, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_up.6.weight", "offset": 11101290496, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_gate.7.weight", "offset": 11120558080, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_down.7.weight", "offset": 11139825664, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.22.ffn_up.7.weight", "offset": 11159093248, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.22.attn_norm.weight", "offset": 11178360832, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 11178377216, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.23.ffn_gate.0.weight", "offset": 11178393600, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.23.ffn_down.0.weight", "offset": 11197661184, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_up.0.weight", "offset": 11216928768, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_gate.1.weight", "offset": 11236196352, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_down.1.weight", "offset": 11255463936, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_up.1.weight", "offset": 11274731520, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_gate.2.weight", "offset": 11293999104, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_down.2.weight", "offset": 11313266688, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_up.2.weight", "offset": 11332534272, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_gate.3.weight", "offset": 11351801856, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_down.3.weight", "offset": 11371069440, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_up.3.weight", "offset": 11390337024, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_gate.4.weight", "offset": 11409604608, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_down.4.weight", "offset": 11428872192, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_up.4.weight", "offset": 11448139776, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_gate.5.weight", "offset": 11467407360, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_down.5.weight", "offset": 11486674944, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_up.5.weight", "offset": 11505942528, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_gate.6.weight", "offset": 11525210112, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 11544477696, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.23.attn_k.weight", "offset": 11544543232, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 11548999680, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.27.attn_n", "offset": 11556208640, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.23.attn_v.weight", "offset": 11561713664, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.23.ffn_down.6.weight", "offset": 11566170112, "shape": [ 14336, 4096 ], "size": 4456448, "type": 10 }, { "name": "blk.23.ffn_up.6.weight", "offset": 11585437696, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_gate.7.weight", "offset": 11604705280, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_down.7.weight", "offset": 11623972864, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.23.ffn_up.7.weight", "offset": 11643240448, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.23.attn_norm.weight", "offset": 11662508032, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 11662524416, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.ffn_gate.0.weight", "offset": 11662540800, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.24.ffn_down.0.weight", "offset": 11681808384, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_up.0.weight", "offset": 11701075968, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_gate.1.weight", "offset": 11720343552, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_down.1.weight", "offset": 11739611136, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_up.1.weight", "offset": 11758878720, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_gate.2.weight", "offset": 11778146304, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_down.2.weight", "offset": 11797413888, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_up.2.weight", "offset": 11816681472, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_gate.3.weight", "offset": 11835949056, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_down.3.weight", "offset": 11855216640, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_up.3.weight", "offset": 11874484224, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_gate.4.weight", "offset": 11893751808, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_down.4.weight", "offset": 11913019392, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_up.4.weight", "offset": 11932286976, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_gate.5.weight", "offset": 11951554560, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_down.5.weight", "offset": 11970822144, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_up.5.weight", "offset": 11990089728, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_gate.6.weight", "offset": 12009357312, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_down.6.weight", "offset": 12028624896, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_up.6.weight", "offset": 12047892480, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_gate.7.weight", "offset": 12067160064, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_down.7.weight", "offset": 12086427648, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_up.7.weight", "offset": 12105695232, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 12124962816, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 12125028352, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 12125044736, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 12125061120, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 12129517568, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.24.attn_q.weight", "offset": 12136726528, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.24.attn_v.weight", "offset": 12142231552, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.25.ffn_gate.0.weight", "offset": 12146688000, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.25.ffn_down.0.weight", "offset": 12165955584, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_up.0.weight", "offset": 12185223168, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_gate.1.weight", "offset": 12204490752, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_down.1.weight", "offset": 12223758336, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_up.1.weight", "offset": 12243025920, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_gate.2.weight", "offset": 12262293504, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_down.2.weight", "offset": 12281561088, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_up.2.weight", "offset": 12300828672, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_gate.3.weight", "offset": 12320096256, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_down.3.weight", "offset": 12339363840, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_up.3.weight", "offset": 12358631424, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 12377899008, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.25.attn_k.weight", "offset": 12377964544, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 12382420992, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.25.attn_q.weight", "offset": 12389629952, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.25.attn_v.weight", "offset": 12395134976, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.25.ffn_gate.4.weight", "offset": 12399591424, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.25.ffn_down.4.weight", "offset": 12418859008, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_up.4.weight", "offset": 12438126592, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_gate.5.weight", "offset": 12457394176, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_down.5.weight", "offset": 12476661760, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_up.5.weight", "offset": 12495929344, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_gate.6.weight", "offset": 12515196928, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_down.6.weight", "offset": 12534464512, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_up.6.weight", "offset": 12553732096, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_gate.7.weight", "offset": 12572999680, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_down.7.weight", "offset": 12592267264, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.25.ffn_up.7.weight", "offset": 12611534848, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.25.attn_norm.weight", "offset": 12630802432, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 12630818816, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.ffn_gate.0.weight", "offset": 12630835200, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.26.ffn_down.0.weight", "offset": 12650102784, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_up.0.weight", "offset": 12669370368, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_gate.1.weight", "offset": 12688637952, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_down.1.weight", "offset": 12707905536, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_up.1.weight", "offset": 12727173120, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_gate.2.weight", "offset": 12746440704, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_down.2.weight", "offset": 12765708288, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_up.2.weight", "offset": 12784975872, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_gate.3.weight", "offset": 12804243456, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_down.3.weight", "offset": 12823511040, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_up.3.weight", "offset": 12842778624, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_gate.4.weight", "offset": 12862046208, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_down.4.weight", "offset": 12881313792, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_up.4.weight", "offset": 12900581376, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_gate.5.weight", "offset": 12919848960, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_down.5.weight", "offset": 12939116544, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_up.5.weight", "offset": 12958384128, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_gate.6.weight", "offset": 12977651712, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_down.6.weight", "offset": 12996919296, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_up.6.weight", "offset": 13016186880, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_gate.7.weight", "offset": 13035454464, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_down.7.weight", "offset": 13054722048, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_up.7.weight", "offset": 13073989632, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 13093257216, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 13093322752, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 13093339136, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 13093355520, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 13097811968, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.26.attn_q.weight", "offset": 13105020928, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.26.attn_v.weight", "offset": 13110525952, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.27.ffn_gate.0.weight", "offset": 13114982400, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.27.ffn_down.0.weight", "offset": 13134249984, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_up.0.weight", "offset": 13153517568, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_gate.1.weight", "offset": 13172785152, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_down.1.weight", "offset": 13192052736, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 13211320320, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.27.attn_k.weight", "offset": 13211385856, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 13215842304, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.27.attn_q.weight", "offset": 13223051264, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.27.attn_v.weight", "offset": 13228556288, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.27.ffn_up.1.weight", "offset": 13233012736, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.27.ffn_gate.2.weight", "offset": 13252280320, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_down.2.weight", "offset": 13271547904, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_up.2.weight", "offset": 13290815488, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_gate.3.weight", "offset": 13310083072, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_down.3.weight", "offset": 13329350656, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_up.3.weight", "offset": 13348618240, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_gate.4.weight", "offset": 13367885824, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_down.4.weight", "offset": 13387153408, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_up.4.weight", "offset": 13406420992, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_gate.5.weight", "offset": 13425688576, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_down.5.weight", "offset": 13444956160, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_up.5.weight", "offset": 13464223744, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_gate.6.weight", "offset": 13483491328, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_down.6.weight", "offset": 13502758912, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_up.6.weight", "offset": 13522026496, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_gate.7.weight", "offset": 13541294080, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_down.7.weight", "offset": 13560561664, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.27.ffn_up.7.weight", "offset": 13579829248, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.27.attn_norm.weight", "offset": 13599096832, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 13599113216, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.28.ffn_gate.0.weight", "offset": 13599129600, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.28.ffn_down.0.weight", "offset": 13618397184, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_up.0.weight", "offset": 13637664768, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_gate.1.weight", "offset": 13656932352, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_down.1.weight", "offset": 13676199936, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_up.1.weight", "offset": 13695467520, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_gate.2.weight", "offset": 13714735104, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_down.2.weight", "offset": 13734002688, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_up.2.weight", "offset": 13753270272, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_gate.3.weight", "offset": 13772537856, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_down.3.weight", "offset": 13791805440, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_up.3.weight", "offset": 13811073024, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_gate.4.weight", "offset": 13830340608, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_down.4.weight", "offset": 13849608192, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_up.4.weight", "offset": 13868875776, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_gate.5.weight", "offset": 13888143360, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_down.5.weight", "offset": 13907410944, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_up.5.weight", "offset": 13926678528, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_gate.6.weight", "offset": 13945946112, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_down.6.weight", "offset": 13965213696, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_up.6.weight", "offset": 13984481280, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_gate.7.weight", "offset": 14003748864, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 14023016448, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.28.attn_k.weight", "offset": 14023081984, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 14027538432, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.28.attn_q.weight", "offset": 14034747392, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.28.attn_v.weight", "offset": 14040252416, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.28.ffn_down.7.weight", "offset": 14044708864, "shape": [ 14336, 4096 ], "size": 4456448, "type": 10 }, { "name": "blk.28.ffn_up.7.weight", "offset": 14063976448, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.28.attn_norm.weight", "offset": 14083244032, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 14083260416, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.ffn_gate.0.weight", "offset": 14083276800, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.29.ffn_down.0.weight", "offset": 14102544384, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_up.0.weight", "offset": 14121811968, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_gate.1.weight", "offset": 14141079552, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_down.1.weight", "offset": 14160347136, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_up.1.weight", "offset": 14179614720, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_gate.2.weight", "offset": 14198882304, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_down.2.weight", "offset": 14218149888, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_up.2.weight", "offset": 14237417472, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_gate.3.weight", "offset": 14256685056, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_down.3.weight", "offset": 14275952640, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_up.3.weight", "offset": 14295220224, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_gate.4.weight", "offset": 14314487808, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_down.4.weight", "offset": 14333755392, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_up.4.weight", "offset": 14353022976, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_gate.5.weight", "offset": 14372290560, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_down.5.weight", "offset": 14391558144, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_up.5.weight", "offset": 14410825728, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_gate.6.weight", "offset": 14430093312, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_down.6.weight", "offset": 14449360896, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_up.6.weight", "offset": 14468628480, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_gate.7.weight", "offset": 14487896064, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_down.7.weight", "offset": 14507163648, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_up.7.weight", "offset": 14526431232, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 14545698816, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.29.attn_norm.weight", "offset": 14545764352, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 14545780736, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 14545797120, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 14550253568, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.29.attn_q.weight", "offset": 14557462528, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.29.attn_v.weight", "offset": 14562967552, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "blk.30.ffn_gate.0.weight", "offset": 14567424000, "shape": [ 4096, 14336 ], "size": 4456448, "type": 10 }, { "name": "blk.30.ffn_down.0.weight", "offset": 14586691584, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_up.0.weight", "offset": 14605959168, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_gate.1.weight", "offset": 14625226752, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_down.1.weight", "offset": 14644494336, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_up.1.weight", "offset": 14663761920, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_gate.2.weight", "offset": 14683029504, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_down.2.weight", "offset": 14702297088, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_up.2.weight", "offset": 14721564672, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_gate.3.weight", "offset": 14740832256, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_down.3.weight", "offset": 14760099840, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_up.3.weight", "offset": 14779367424, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_gate.4.weight", "offset": 14798635008, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_down.4.weight", "offset": 14817902592, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_up.4.weight", "offset": 14837170176, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 14856437760, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.30.attn_k.weight", "offset": 14856503296, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 14860959744, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.30.attn_q.weight", "offset": 14868168704, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.30.attn_v.weight", "offset": 14873673728, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "output.weight", "offset": 14878130176, "shape": [ 4096, 32000 ], "size": 4456448, "type": 14 }, { "name": "blk.30.ffn_gate.5.weight", "offset": 14985650176, "shape": [ 4096, 14336 ], "size": 107520000, "type": 10 }, { "name": "blk.30.ffn_down.5.weight", "offset": 15004917760, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_up.5.weight", "offset": 15024185344, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_gate.6.weight", "offset": 15043452928, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_down.6.weight", "offset": 15062720512, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_up.6.weight", "offset": 15081988096, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_gate.7.weight", "offset": 15101255680, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_down.7.weight", "offset": 15120523264, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.30.ffn_up.7.weight", "offset": 15139790848, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.30.attn_norm.weight", "offset": 15159058432, "shape": [ 4096 ], "size": 19267584, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 15159074816, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.ffn_gate.0.weight", "offset": 15159091200, "shape": [ 4096, 14336 ], "size": 16384, "type": 10 }, { "name": "blk.31.ffn_down.0.weight", "offset": 15178358784, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_up.0.weight", "offset": 15197626368, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_gate.1.weight", "offset": 15216893952, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_down.1.weight", "offset": 15236161536, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_up.1.weight", "offset": 15255429120, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_gate.2.weight", "offset": 15274696704, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_down.2.weight", "offset": 15293964288, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_up.2.weight", "offset": 15313231872, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_gate.3.weight", "offset": 15332499456, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_down.3.weight", "offset": 15351767040, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_up.3.weight", "offset": 15371034624, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_gate.4.weight", "offset": 15390302208, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_down.4.weight", "offset": 15409569792, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_up.4.weight", "offset": 15428837376, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_gate.5.weight", "offset": 15448104960, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_down.5.weight", "offset": 15467372544, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_up.5.weight", "offset": 15486640128, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_gate.6.weight", "offset": 15505907712, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_down.6.weight", "offset": 15525175296, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_up.6.weight", "offset": 15544442880, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_gate.7.weight", "offset": 15563710464, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_down.7.weight", "offset": 15582978048, "shape": [ 14336, 4096 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_up.7.weight", "offset": 15602245632, "shape": [ 4096, 14336 ], "size": 19267584, "type": 10 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 15621513216, "shape": [ 4096, 8 ], "size": 19267584, "type": 1 }, { "name": "blk.31.attn_norm.weight", "offset": 15621578752, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 15621595136, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 15621611520, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 15626067968, "shape": [ 4096, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.31.attn_q.weight", "offset": 15633276928, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.31.attn_v.weight", "offset": 15638781952, "shape": [ 4096, 1024 ], "size": 5505024, "type": 8 }, { "name": "output_norm.weight", "offset": 15643238400, "shape": [ 4096 ], "size": 4456448, "type": 0 } ], "version": 3 }