mixtral

{ "metadata": { "general.architecture": "llama", "general.file_type": 14, "general.name": "mistralai", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 32, "llama.context_length": 32768, "llama.embedding_length": 4096, "llama.expert_count": 8, "llama.expert_used_count": 2, "llama.feed_forward_length": 14336, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.merges": "... (58980 values)", "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32000 values)", "tokenizer.ggml.token_type": "... (32000 values)", "tokenizer.ggml.tokens": "... (32000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 46702792704, "tensors": [ { "name": "blk.0.ffn_gate.0.weight", "offset": 73728000, "shape": [ 4096, 14336 ], "size": 73728000, "type": 12 }, { "name": "blk.0.ffn_down.0.weight", "offset": 106758144, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_up.0.weight", "offset": 139788288, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_gate.1.weight", "offset": 172818432, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_down.1.weight", "offset": 205848576, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_up.1.weight", "offset": 238878720, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_gate.2.weight", "offset": 271908864, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_down.2.weight", "offset": 304939008, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_up.2.weight", "offset": 337969152, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_gate.3.weight", "offset": 370999296, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_down.3.weight", "offset": 404029440, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_up.3.weight", "offset": 437059584, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_gate.4.weight", "offset": 470089728, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_down.4.weight", "offset": 503119872, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_up.4.weight", "offset": 536150016, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_gate.5.weight", "offset": 569180160, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_down.5.weight", "offset": 602210304, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_up.5.weight", "offset": 635240448, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_gate.6.weight", "offset": 668270592, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_down.6.weight", "offset": 701300736, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_up.6.weight", "offset": 734330880, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_gate.7.weight", "offset": 767361024, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_down.7.weight", "offset": 800391168, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_up.7.weight", "offset": 833421312, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.0.ffn_gate_inp.weight", "offset": 866451456, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.0.attn_norm.weight", "offset": 866516992, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 866533376, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 866549760, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 871006208, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 880443392, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.0.attn_v.weight", "offset": 889880576, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.1.ffn_gate.0.weight", "offset": 894337024, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.1.ffn_down.0.weight", "offset": 927367168, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_up.0.weight", "offset": 960397312, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_gate.1.weight", "offset": 993427456, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_down.1.weight", "offset": 1026457600, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_up.1.weight", "offset": 1059487744, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_gate.2.weight", "offset": 1092517888, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_down.2.weight", "offset": 1125548032, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_up.2.weight", "offset": 1158578176, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_gate.3.weight", "offset": 1191608320, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_down.3.weight", "offset": 1224638464, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_up.3.weight", "offset": 1257668608, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_gate.4.weight", "offset": 1290698752, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_down.4.weight", "offset": 1323728896, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1356759040, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.1.attn_k.weight", "offset": 1356824576, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 1361281024, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 1370718208, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.1.attn_v.weight", "offset": 1380155392, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.1.ffn_up.4.weight", "offset": 1384611840, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.1.ffn_gate.5.weight", "offset": 1417641984, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_down.5.weight", "offset": 1450672128, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_up.5.weight", "offset": 1483702272, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_gate.6.weight", "offset": 1516732416, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_down.6.weight", "offset": 1549762560, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_up.6.weight", "offset": 1582792704, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_gate.7.weight", "offset": 1615822848, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_down.7.weight", "offset": 1648852992, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.1.ffn_up.7.weight", "offset": 1681883136, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.1.attn_norm.weight", "offset": 1714913280, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 1714929664, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.ffn_gate.0.weight", "offset": 1714946048, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.2.ffn_down.0.weight", "offset": 1747976192, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_up.0.weight", "offset": 1781006336, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_gate.1.weight", "offset": 1814036480, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_down.1.weight", "offset": 1847066624, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_up.1.weight", "offset": 1880096768, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_gate.2.weight", "offset": 1913126912, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_down.2.weight", "offset": 1946157056, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_up.2.weight", "offset": 1979187200, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_gate.3.weight", "offset": 2012217344, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_down.3.weight", "offset": 2045247488, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_up.3.weight", "offset": 2078277632, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_gate.4.weight", "offset": 2111307776, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_down.4.weight", "offset": 2144337920, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_up.4.weight", "offset": 2177368064, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_gate.5.weight", "offset": 2210398208, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_down.5.weight", "offset": 2243428352, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_up.5.weight", "offset": 2276458496, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_gate.6.weight", "offset": 2309488640, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_down.6.weight", "offset": 2342518784, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_up.6.weight", "offset": 2375548928, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_gate.7.weight", "offset": 2408579072, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_down.7.weight", "offset": 2441609216, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_up.7.weight", "offset": 2474639360, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 2507669504, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 2507735040, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 2507751424, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 2507767808, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 2512224256, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 2521661440, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.2.attn_v.weight", "offset": 2531098624, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.3.ffn_gate.0.weight", "offset": 2535555072, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.3.ffn_down.0.weight", "offset": 2568585216, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_up.0.weight", "offset": 2601615360, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_gate.1.weight", "offset": 2634645504, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_down.1.weight", "offset": 2667675648, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_up.1.weight", "offset": 2700705792, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_gate.2.weight", "offset": 2733735936, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 2766766080, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.3.attn_k.weight", "offset": 2766831616, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 2771288064, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 2780725248, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.3.attn_v.weight", "offset": 2790162432, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.3.ffn_down.2.weight", "offset": 2794618880, "shape": [ 14336, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.3.ffn_up.2.weight", "offset": 2827649024, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_gate.3.weight", "offset": 2860679168, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_down.3.weight", "offset": 2893709312, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_up.3.weight", "offset": 2926739456, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_gate.4.weight", "offset": 2959769600, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_down.4.weight", "offset": 2992799744, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_up.4.weight", "offset": 3025829888, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_gate.5.weight", "offset": 3058860032, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_down.5.weight", "offset": 3091890176, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_up.5.weight", "offset": 3124920320, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_gate.6.weight", "offset": 3157950464, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_down.6.weight", "offset": 3190980608, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_up.6.weight", "offset": 3224010752, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_gate.7.weight", "offset": 3257040896, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_down.7.weight", "offset": 3290071040, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.3.ffn_up.7.weight", "offset": 3323101184, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 3356131328, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 3356147712, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.ffn_gate.0.weight", "offset": 3356164096, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.4.ffn_down.0.weight", "offset": 3389194240, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_up.0.weight", "offset": 3422224384, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_gate.1.weight", "offset": 3455254528, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_down.1.weight", "offset": 3488284672, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_up.1.weight", "offset": 3521314816, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_gate.2.weight", "offset": 3554344960, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_down.2.weight", "offset": 3587375104, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_up.2.weight", "offset": 3620405248, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_gate.3.weight", "offset": 3653435392, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_down.3.weight", "offset": 3686465536, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_up.3.weight", "offset": 3719495680, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_gate.4.weight", "offset": 3752525824, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_down.4.weight", "offset": 3785555968, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_up.4.weight", "offset": 3818586112, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_gate.5.weight", "offset": 3851616256, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_down.5.weight", "offset": 3884646400, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_up.5.weight", "offset": 3917676544, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_gate.6.weight", "offset": 3950706688, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_down.6.weight", "offset": 3983736832, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_up.6.weight", "offset": 4016766976, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_gate.7.weight", "offset": 4049797120, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_down.7.weight", "offset": 4082827264, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_up.7.weight", "offset": 4115857408, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 4148887552, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 4148953088, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 4148969472, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 4148985856, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 4153442304, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 4162879488, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.4.attn_v.weight", "offset": 4172316672, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 4176773120, "shape": [ 4096, 8 ], "size": 4456448, "type": 1 }, { "name": "blk.5.attn_k.weight", "offset": 4176838656, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 4181295104, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 4190732288, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.5.attn_v.weight", "offset": 4200169472, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.5.ffn_gate.0.weight", "offset": 4204625920, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.5.ffn_down.0.weight", "offset": 4237656064, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_up.0.weight", "offset": 4270686208, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_gate.1.weight", "offset": 4303716352, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_down.1.weight", "offset": 4336746496, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_up.1.weight", "offset": 4369776640, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_gate.2.weight", "offset": 4402806784, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_down.2.weight", "offset": 4435836928, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_up.2.weight", "offset": 4468867072, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_gate.3.weight", "offset": 4501897216, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_down.3.weight", "offset": 4534927360, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_up.3.weight", "offset": 4567957504, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_gate.4.weight", "offset": 4600987648, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_down.4.weight", "offset": 4634017792, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_up.4.weight", "offset": 4667047936, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_gate.5.weight", "offset": 4700078080, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_down.5.weight", "offset": 4733108224, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_up.5.weight", "offset": 4766138368, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_gate.6.weight", "offset": 4799168512, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_down.6.weight", "offset": 4832198656, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_up.6.weight", "offset": 4865228800, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_gate.7.weight", "offset": 4898258944, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_down.7.weight", "offset": 4931289088, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.5.ffn_up.7.weight", "offset": 4964319232, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 4997349376, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 4997365760, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.6.ffn_gate.0.weight", "offset": 4997382144, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.6.ffn_down.0.weight", "offset": 5030412288, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_up.0.weight", "offset": 5063442432, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_gate.1.weight", "offset": 5096472576, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_down.1.weight", "offset": 5129502720, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_up.1.weight", "offset": 5162532864, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_gate.2.weight", "offset": 5195563008, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_down.2.weight", "offset": 5228593152, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_up.2.weight", "offset": 5261623296, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_gate.3.weight", "offset": 5294653440, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_down.3.weight", "offset": 5327683584, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_up.3.weight", "offset": 5360713728, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_gate.4.weight", "offset": 5393743872, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_down.4.weight", "offset": 5426774016, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_up.4.weight", "offset": 5459804160, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_gate.5.weight", "offset": 5492834304, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_down.5.weight", "offset": 5525864448, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 5558894592, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.6.attn_k.weight", "offset": 5558960128, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 5563416576, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 5572853760, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.6.attn_v.weight", "offset": 5582290944, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.6.ffn_up.5.weight", "offset": 5586747392, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.6.ffn_gate.6.weight", "offset": 5619777536, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_down.6.weight", "offset": 5652807680, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_up.6.weight", "offset": 5685837824, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_gate.7.weight", "offset": 5718867968, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_down.7.weight", "offset": 5751898112, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.6.ffn_up.7.weight", "offset": 5784928256, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 5817958400, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 5817974784, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.ffn_gate.0.weight", "offset": 5817991168, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.7.ffn_down.0.weight", "offset": 5851021312, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_up.0.weight", "offset": 5884051456, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_gate.1.weight", "offset": 5917081600, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_down.1.weight", "offset": 5950111744, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_up.1.weight", "offset": 5983141888, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_gate.2.weight", "offset": 6016172032, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_down.2.weight", "offset": 6049202176, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_up.2.weight", "offset": 6082232320, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_gate.3.weight", "offset": 6115262464, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_down.3.weight", "offset": 6148292608, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_up.3.weight", "offset": 6181322752, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_gate.4.weight", "offset": 6214352896, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_down.4.weight", "offset": 6247383040, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_up.4.weight", "offset": 6280413184, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_gate.5.weight", "offset": 6313443328, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_down.5.weight", "offset": 6346473472, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_up.5.weight", "offset": 6379503616, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_gate.6.weight", "offset": 6412533760, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_down.6.weight", "offset": 6445563904, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_up.6.weight", "offset": 6478594048, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_gate.7.weight", "offset": 6511624192, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_down.7.weight", "offset": 6544654336, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_up.7.weight", "offset": 6577684480, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 6610714624, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.7.attn_norm.weight", "offset": 6610780160, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 6610796544, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 6610812928, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 6615269376, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 6624706560, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.7.attn_v.weight", "offset": 6634143744, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.8.ffn_gate.0.weight", "offset": 6638600192, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.8.ffn_down.0.weight", "offset": 6671630336, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_up.0.weight", "offset": 6704660480, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_gate.1.weight", "offset": 6737690624, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_down.1.weight", "offset": 6770720768, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_up.1.weight", "offset": 6803750912, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_gate.2.weight", "offset": 6836781056, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_down.2.weight", "offset": 6869811200, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_up.2.weight", "offset": 6902841344, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_gate.3.weight", "offset": 6935871488, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 6968901632, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.8.attn_k.weight", "offset": 6968967168, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 6973423616, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 6982860800, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.8.attn_v.weight", "offset": 6992297984, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.10.ffn_gate.0.weight", "offset": 6996754432, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.10.ffn_down.0.weight", "offset": 7029784576, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_up.0.weight", "offset": 7062814720, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 7095844864, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.10.attn_k.weight", "offset": 7095910400, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 7100366848, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.10.attn_q.weight", "offset": 7109804032, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.10.attn_v.weight", "offset": 7119241216, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.8.ffn_down.3.weight", "offset": 7123697664, "shape": [ 14336, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.8.ffn_up.3.weight", "offset": 7156727808, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_gate.4.weight", "offset": 7189757952, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_down.4.weight", "offset": 7222788096, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_up.4.weight", "offset": 7255818240, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_gate.5.weight", "offset": 7288848384, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_down.5.weight", "offset": 7321878528, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_up.5.weight", "offset": 7354908672, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_gate.6.weight", "offset": 7387938816, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_down.6.weight", "offset": 7420968960, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_up.6.weight", "offset": 7453999104, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_gate.7.weight", "offset": 7487029248, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_down.7.weight", "offset": 7520059392, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.8.ffn_up.7.weight", "offset": 7553089536, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 7586119680, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 7586136064, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.ffn_gate.0.weight", "offset": 7586152448, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.9.ffn_down.0.weight", "offset": 7619182592, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_up.0.weight", "offset": 7652212736, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_gate.1.weight", "offset": 7685242880, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_down.1.weight", "offset": 7718273024, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_up.1.weight", "offset": 7751303168, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_gate.2.weight", "offset": 7784333312, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_down.2.weight", "offset": 7817363456, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_up.2.weight", "offset": 7850393600, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_gate.3.weight", "offset": 7883423744, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_down.3.weight", "offset": 7916453888, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_up.3.weight", "offset": 7949484032, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_gate.4.weight", "offset": 7982514176, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_down.4.weight", "offset": 8015544320, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_up.4.weight", "offset": 8048574464, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_gate.5.weight", "offset": 8081604608, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_down.5.weight", "offset": 8114634752, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_up.5.weight", "offset": 8147664896, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_gate.6.weight", "offset": 8180695040, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_down.6.weight", "offset": 8213725184, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_up.6.weight", "offset": 8246755328, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_gate.7.weight", "offset": 8279785472, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_down.7.weight", "offset": 8312815616, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_up.7.weight", "offset": 8345845760, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 8378875904, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 8378941440, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 8378957824, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 8378974208, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 8383430656, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 8392867840, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.9.attn_v.weight", "offset": 8402305024, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.10.ffn_gate.1.weight", "offset": 8406761472, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.10.ffn_down.1.weight", "offset": 8439791616, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_up.1.weight", "offset": 8472821760, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_gate.2.weight", "offset": 8505851904, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_down.2.weight", "offset": 8538882048, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_up.2.weight", "offset": 8571912192, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_gate.3.weight", "offset": 8604942336, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_down.3.weight", "offset": 8637972480, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_up.3.weight", "offset": 8671002624, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_gate.4.weight", "offset": 8704032768, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_down.4.weight", "offset": 8737062912, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_up.4.weight", "offset": 8770093056, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_gate.5.weight", "offset": 8803123200, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_down.5.weight", "offset": 8836153344, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_up.5.weight", "offset": 8869183488, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_gate.6.weight", "offset": 8902213632, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "15.ffn_gate.0.weight\u0002\u0000\u0000\u0000", "offset": 8935243776, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_up.6.weight", "offset": 8968273920, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_gate.7.weight", "offset": 9001304064, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_down.7.weight", "offset": 9034334208, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.10.ffn_up.7.weight", "offset": 9067364352, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 9100394496, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 9100410880, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.11.ffn_gate.0.weight", "offset": 9100427264, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.11.ffn_down.0.weight", "offset": 9133457408, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_up.0.weight", "offset": 9166487552, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_gate.1.weight", "offset": 9199517696, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_down.1.weight", "offset": 9232547840, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_up.1.weight", "offset": 9265577984, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_gate.2.weight", "offset": 9298608128, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_down.2.weight", "offset": 9331638272, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_up.2.weight", "offset": 9364668416, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_gate.3.weight", "offset": 9397698560, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_down.3.weight", "offset": 9430728704, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_up.3.weight", "offset": 9463758848, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_gate.4.weight", "offset": 9496788992, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_down.4.weight", "offset": 9529819136, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_up.4.weight", "offset": 9562849280, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_gate.5.weight", "offset": 9595879424, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_down.5.weight", "offset": 9628909568, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_up.5.weight", "offset": 9661939712, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_gate.6.weight", "offset": 9694969856, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_down.6.weight", "offset": 9728000000, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 9761030144, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.11.attn_k.weight", "offset": 9761095680, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 9765552128, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.11.attn_q.weight", "offset": 9774989312, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.11.attn_v.weight", "offset": 9784426496, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.11.ffn_up.6.weight", "offset": 9788882944, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.11.ffn_gate.7.weight", "offset": 9821913088, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_down.7.weight", "offset": 9854943232, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.11.ffn_up.7.weight", "offset": 9887973376, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 9921003520, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 9921019904, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.ffn_gate.0.weight", "offset": 9921036288, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.12.ffn_down.0.weight", "offset": 9954066432, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_up.0.weight", "offset": 9987096576, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_gate.1.weight", "offset": 10020126720, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_down.1.weight", "offset": 10053156864, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_up.1.weight", "offset": 10086187008, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_gate.2.weight", "offset": 10119217152, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_down.2.weight", "offset": 10152247296, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_up.2.weight", "offset": 10185277440, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_gate.3.weight", "offset": 10218307584, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_down.3.weight", "offset": 10251337728, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_up.3.weight", "offset": 10284367872, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_gate.4.weight", "offset": 10317398016, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_down.4.weight", "offset": 10350428160, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_up.4.weight", "offset": 10383458304, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_gate.5.weight", "offset": 10416488448, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_down.5.weight", "offset": 10449518592, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_up.5.weight", "offset": 10482548736, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_gate.6.weight", "offset": 10515578880, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_down.6.weight", "offset": 10548609024, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_up.6.weight", "offset": 10581639168, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_gate.7.weight", "offset": 10614669312, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_down.7.weight", "offset": 10647699456, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_up.7.weight", "offset": 10680729600, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 10713759744, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 10713825280, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 10713841664, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 10713858048, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 10718314496, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.12.attn_q.weight", "offset": 10727751680, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.12.attn_v.weight", "offset": 10737188864, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.13.ffn_gate.0.weight", "offset": 10741645312, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.13.ffn_down.0.weight", "offset": 10774675456, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_up.0.weight", "offset": 10807705600, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_gate.1.weight", "offset": 10840735744, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_down.1.weight", "offset": 10873765888, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_up.1.weight", "offset": 10906796032, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_gate.2.weight", "offset": 10939826176, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_down.2.weight", "offset": 10972856320, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_up.2.weight", "offset": 11005886464, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_gate.3.weight", "offset": 11038916608, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_down.3.weight", "offset": 11071946752, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_up.3.weight", "offset": 11104976896, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_gate.4.weight", "offset": 11138007040, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 11171037184, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.13.attn_k.weight", "offset": 11171102720, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 11175559168, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.13.attn_q.weight", "offset": 11184996352, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.13.attn_v.weight", "offset": 11194433536, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.13.ffn_down.4.weight", "offset": 11198889984, "shape": [ 14336, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.13.ffn_up.4.weight", "offset": 11231920128, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_gate.5.weight", "offset": 11264950272, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_down.5.weight", "offset": 11297980416, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_up.5.weight", "offset": 11331010560, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_gate.6.weight", "offset": 11364040704, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_down.6.weight", "offset": 11397070848, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_up.6.weight", "offset": 11430100992, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_gate.7.weight", "offset": 11463131136, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_down.7.weight", "offset": 11496161280, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.13.ffn_up.7.weight", "offset": 11529191424, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 11562221568, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 11562237952, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.ffn_gate.0.weight", "offset": 11562254336, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.14.ffn_down.0.weight", "offset": 11595284480, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_up.0.weight", "offset": 11628314624, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_gate.1.weight", "offset": 11661344768, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_down.1.weight", "offset": 11694374912, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_up.1.weight", "offset": 11727405056, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_gate.2.weight", "offset": 11760435200, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_down.2.weight", "offset": 11793465344, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_up.2.weight", "offset": 11826495488, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_gate.3.weight", "offset": 11859525632, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_down.3.weight", "offset": 11892555776, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_up.3.weight", "offset": 11925585920, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_gate.4.weight", "offset": 11958616064, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_down.4.weight", "offset": 11991646208, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_up.4.weight", "offset": 12024676352, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_gate.5.weight", "offset": 12057706496, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_down.5.weight", "offset": 12090736640, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_up.5.weight", "offset": 12123766784, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_gate.6.weight", "offset": 12156796928, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_down.6.weight", "offset": 12189827072, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_up.6.weight", "offset": 12222857216, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_gate.7.weight", "offset": 12255887360, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_down.7.weight", "offset": 12288917504, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_up.7.weight", "offset": 12321947648, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 12354977792, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 12355043328, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 12355059712, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 12355076096, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 12359532544, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 12368969728, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.14.attn_v.weight", "offset": 12378406912, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.15.ffn_gate.0.weight", "offset": 12382863360, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.15.ffn_down.0.weight", "offset": 12415893504, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_up.0.weight", "offset": 12448923648, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_gate.1.weight", "offset": 12481953792, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_down.1.weight", "offset": 12514983936, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_up.1.weight", "offset": 12548014080, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 12581044224, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.15.attn_k.weight", "offset": 12581109760, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 12585566208, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 12595003392, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.15.attn_v.weight", "offset": 12604440576, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.15.ffn_gate.2.weight", "offset": 12608897024, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.15.ffn_down.2.weight", "offset": 12641927168, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_up.2.weight", "offset": 12674957312, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_gate.3.weight", "offset": 12707987456, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_down.3.weight", "offset": 12741017600, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_up.3.weight", "offset": 12774047744, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_gate.4.weight", "offset": 12807077888, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_down.4.weight", "offset": 12840108032, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_up.4.weight", "offset": 12873138176, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_gate.5.weight", "offset": 12906168320, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_down.5.weight", "offset": 12939198464, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_up.5.weight", "offset": 12972228608, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_gate.6.weight", "offset": 13005258752, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_down.6.weight", "offset": 13038288896, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_up.6.weight", "offset": 13071319040, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_gate.7.weight", "offset": 13104349184, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_down.7.weight", "offset": 13137379328, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.15.ffn_up.7.weight", "offset": 13170409472, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 13203439616, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 13203456000, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.16.ffn_gate.0.weight", "offset": 13203472384, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.16.ffn_down.0.weight", "offset": 13236502528, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_up.0.weight", "offset": 13269532672, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_gate.1.weight", "offset": 13302562816, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_down.1.weight", "offset": 13335592960, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_up.1.weight", "offset": 13368623104, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_gate.2.weight", "offset": 13401653248, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_down.2.weight", "offset": 13434683392, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_up.2.weight", "offset": 13467713536, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_gate.3.weight", "offset": 13500743680, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_down.3.weight", "offset": 13533773824, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_up.3.weight", "offset": 13566803968, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_gate.4.weight", "offset": 13599834112, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_down.4.weight", "offset": 13632864256, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_up.4.weight", "offset": 13665894400, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_gate.5.weight", "offset": 13698924544, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_down.5.weight", "offset": 13731954688, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_up.5.weight", "offset": 13764984832, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_gate.6.weight", "offset": 13798014976, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_down.6.weight", "offset": 13831045120, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_up.6.weight", "offset": 13864075264, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_gate.7.weight", "offset": 13897105408, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_down.7.weight", "offset": 13930135552, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 13963165696, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.16.attn_k.weight", "offset": 13963231232, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 13967687680, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.16.attn_q.weight", "offset": 13977124864, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.16.attn_v.weight", "offset": 13986562048, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.16.ffn_up.7.weight", "offset": 13991018496, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 14024048640, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 14024065024, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.ffn_gate.0.weight", "offset": 14024081408, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.17.ffn_down.0.weight", "offset": 14057111552, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_up.0.weight", "offset": 14090141696, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_gate.1.weight", "offset": 14123171840, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_down.1.weight", "offset": 14156201984, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_up.1.weight", "offset": 14189232128, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_gate.2.weight", "offset": 14222262272, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_down.2.weight", "offset": 14255292416, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_up.2.weight", "offset": 14288322560, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_gate.3.weight", "offset": 14321352704, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_down.3.weight", "offset": 14354382848, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_up.3.weight", "offset": 14387412992, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_gate.4.weight", "offset": 14420443136, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_down.4.weight", "offset": 14453473280, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_up.4.weight", "offset": 14486503424, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_gate.5.weight", "offset": 14519533568, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_down.5.weight", "offset": 14552563712, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_up.5.weight", "offset": 14585593856, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_gate.6.weight", "offset": 14618624000, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_down.6.weight", "offset": 14651654144, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_up.6.weight", "offset": 14684684288, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_gate.7.weight", "offset": 14717714432, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_down.7.weight", "offset": 14750744576, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_up.7.weight", "offset": 14783774720, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 14816804864, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 14816870400, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 14816886784, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 14816903168, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 14821359616, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 14830796800, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.17.attn_v.weight", "offset": 14840233984, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.18.ffn_gate.0.weight", "offset": 14844690432, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.18.ffn_down.0.weight", "offset": 14877720576, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_up.0.weight", "offset": 14910750720, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_gate.1.weight", "offset": 14943780864, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_down.1.weight", "offset": 14976811008, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_up.1.weight", "offset": 15009841152, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_gate.2.weight", "offset": 15042871296, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_down.2.weight", "offset": 15075901440, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_up.2.weight", "offset": 15108931584, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_gate.3.weight", "offset": 15141961728, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_down.3.weight", "offset": 15174991872, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_up.3.weight", "offset": 15208022016, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_gate.4.weight", "offset": 15241052160, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_down.4.weight", "offset": 15274082304, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_up.4.weight", "offset": 15307112448, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_gate.5.weight", "offset": 15340142592, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 15373172736, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.18.attn_k.weight", "offset": 15373238272, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 15377694720, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.18.attn_q.weight", "offset": 15387131904, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.18.attn_v.weight", "offset": 15396569088, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.18.ffn_down.5.weight", "offset": 15401025536, "shape": [ 14336, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.18.ffn_up.5.weight", "offset": 15434055680, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_gate.6.weight", "offset": 15467085824, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_down.6.weight", "offset": 15500115968, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_up.6.weight", "offset": 15533146112, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_gate.7.weight", "offset": 15566176256, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_down.7.weight", "offset": 15599206400, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.18.ffn_up.7.weight", "offset": 15632236544, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 15665266688, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 15665283072, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.ffn_gate.0.weight", "offset": 15665299456, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.19.ffn_down.0.weight", "offset": 15698329600, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_up.0.weight", "offset": 15731359744, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_gate.1.weight", "offset": 15764389888, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_down.1.weight", "offset": 15797420032, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_up.1.weight", "offset": 15830450176, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_gate.2.weight", "offset": 15863480320, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_down.2.weight", "offset": 15896510464, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_up.2.weight", "offset": 15929540608, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_gate.3.weight", "offset": 15962570752, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_down.3.weight", "offset": 15995600896, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_up.3.weight", "offset": 16028631040, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_gate.4.weight", "offset": 16061661184, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_down.4.weight", "offset": 16094691328, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_up.4.weight", "offset": 16127721472, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_gate.5.weight", "offset": 16160751616, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_down.5.weight", "offset": 16193781760, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_up.5.weight", "offset": 16226811904, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_gate.6.weight", "offset": 16259842048, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_down.6.weight", "offset": 16292872192, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_up.6.weight", "offset": 16325902336, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_gate.7.weight", "offset": 16358932480, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_down.7.weight", "offset": 16391962624, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_up.7.weight", "offset": 16424992768, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 16458022912, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.19.attn_norm.weight", "offset": 16458088448, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 16458104832, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 16458121216, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 16462577664, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 16472014848, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.19.attn_v.weight", "offset": 16481452032, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.20.ffn_gate.0.weight", "offset": 16485908480, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.20.ffn_down.0.weight", "offset": 16518938624, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_up.0.weight", "offset": 16551968768, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_gate.1.weight", "offset": 16584998912, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_down.1.weight", "offset": 16618029056, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_up.1.weight", "offset": 16651059200, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_gate.2.weight", "offset": 16684089344, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_down.2.weight", "offset": 16717119488, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_up.2.weight", "offset": 16750149632, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 16783179776, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.20.attn_k.weight", "offset": 16783245312, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 16787701760, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 16797138944, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.20.attn_v.weight", "offset": 16806576128, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.20.ffn_gate.3.weight", "offset": 16811032576, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.20.ffn_down.3.weight", "offset": 16844062720, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_up.3.weight", "offset": 16877092864, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_gate.4.weight", "offset": 16910123008, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_down.4.weight", "offset": 16943153152, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_up.4.weight", "offset": 16976183296, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_gate.5.weight", "offset": 17009213440, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_down.5.weight", "offset": 17042243584, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_up.5.weight", "offset": 17075273728, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_gate.6.weight", "offset": 17108303872, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_down.6.weight", "offset": 17141334016, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_up.6.weight", "offset": 17174364160, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_gate.7.weight", "offset": 17207394304, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_down.7.weight", "offset": 17240424448, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.20.ffn_up.7.weight", "offset": 17273454592, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 17306484736, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 17306501120, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.ffn_gate.0.weight", "offset": 17306517504, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.21.ffn_down.0.weight", "offset": 17339547648, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_up.0.weight", "offset": 17372577792, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_gate.1.weight", "offset": 17405607936, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_down.1.weight", "offset": 17438638080, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_up.1.weight", "offset": 17471668224, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_gate.2.weight", "offset": 17504698368, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_down.2.weight", "offset": 17537728512, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_up.2.weight", "offset": 17570758656, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_gate.3.weight", "offset": 17603788800, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_down.3.weight", "offset": 17636818944, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_up.3.weight", "offset": 17669849088, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_gate.4.weight", "offset": 17702879232, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_down.4.weight", "offset": 17735909376, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_up.4.weight", "offset": 17768939520, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_gate.5.weight", "offset": 17801969664, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_down.5.weight", "offset": 17834999808, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_up.5.weight", "offset": 17868029952, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_gate.6.weight", "offset": 17901060096, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_down.6.weight", "offset": 17934090240, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_up.6.weight", "offset": 17967120384, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_gate.7.weight", "offset": 18000150528, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_down.7.weight", "offset": 18033180672, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_up.7.weight", "offset": 18066210816, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 18099240960, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 18099306496, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 18099322880, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 18099339264, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 18103795712, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 18113232896, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.21.attn_v.weight", "offset": 18122670080, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.22.ffn_gate.0.weight", "offset": 18127126528, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.22.ffn_down.0.weight", "offset": 18160156672, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 18193186816, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.22.attn_k.weight", "offset": 18193252352, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 18197708800, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 18207145984, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.22.attn_v.weight", "offset": 18216583168, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.22.ffn_up.0.weight", "offset": 18221039616, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.22.ffn_gate.1.weight", "offset": 18254069760, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_down.1.weight", "offset": 18287099904, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_up.1.weight", "offset": 18320130048, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_gate.2.weight", "offset": 18353160192, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_down.2.weight", "offset": 18386190336, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_up.2.weight", "offset": 18419220480, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_gate.3.weight", "offset": 18452250624, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_down.3.weight", "offset": 18485280768, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_up.3.weight", "offset": 18518310912, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_gate.4.weight", "offset": 18551341056, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_down.4.weight", "offset": 18584371200, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_up.4.weight", "offset": 18617401344, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_gate.5.weight", "offset": 18650431488, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_down.5.weight", "offset": 18683461632, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_up.5.weight", "offset": 18716491776, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_gate.6.weight", "offset": 18749521920, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_down.6.weight", "offset": 18782552064, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_up.6.weight", "offset": 18815582208, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_gate.7.weight", "offset": 18848612352, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_down.7.weight", "offset": 18881642496, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.22.ffn_up.7.weight", "offset": 18914672640, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 18947702784, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 18947719168, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.23.ffn_gate.0.weight", "offset": 18947735552, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.23.ffn_down.0.weight", "offset": 18980765696, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_up.0.weight", "offset": 19013795840, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_gate.1.weight", "offset": 19046825984, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_down.1.weight", "offset": 19079856128, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_up.1.weight", "offset": 19112886272, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_gate.2.weight", "offset": 19145916416, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_down.2.weight", "offset": 19178946560, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_up.2.weight", "offset": 19211976704, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_gate.3.weight", "offset": 19245006848, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_down.3.weight", "offset": 19278036992, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_up.3.weight", "offset": 19311067136, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_gate.4.weight", "offset": 19344097280, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_down.4.weight", "offset": 19377127424, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_up.4.weight", "offset": 19410157568, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_gate.5.weight", "offset": 19443187712, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_down.5.weight", "offset": 19476217856, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_up.5.weight", "offset": 19509248000, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_gate.6.weight", "offset": 19542278144, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 19575308288, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.23.attn_k.weight", "offset": 19575373824, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 19579830272, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 19589267456, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.23.attn_v.weight", "offset": 19598704640, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.23.ffn_down.6.weight", "offset": 19603161088, "shape": [ 14336, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.23.ffn_up.6.weight", "offset": 19636191232, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_gate.7.weight", "offset": 19669221376, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_down.7.weight", "offset": 19702251520, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.23.ffn_up.7.weight", "offset": 19735281664, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 19768311808, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 19768328192, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.ffn_gate.0.weight", "offset": 19768344576, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.24.ffn_down.0.weight", "offset": 19801374720, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_up.0.weight", "offset": 19834404864, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_gate.1.weight", "offset": 19867435008, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_down.1.weight", "offset": 19900465152, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_up.1.weight", "offset": 19933495296, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_gate.2.weight", "offset": 19966525440, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_down.2.weight", "offset": 19999555584, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_up.2.weight", "offset": 20032585728, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_gate.3.weight", "offset": 20065615872, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_down.3.weight", "offset": 20098646016, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_up.3.weight", "offset": 20131676160, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_gate.4.weight", "offset": 20164706304, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_down.4.weight", "offset": 20197736448, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_up.4.weight", "offset": 20230766592, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_gate.5.weight", "offset": 20263796736, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_down.5.weight", "offset": 20296826880, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_up.5.weight", "offset": 20329857024, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_gate.6.weight", "offset": 20362887168, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_down.6.weight", "offset": 20395917312, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_up.6.weight", "offset": 20428947456, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_gate.7.weight", "offset": 20461977600, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_down.7.weight", "offset": 20495007744, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_up.7.weight", "offset": 20528037888, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 20561068032, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 20561133568, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 20561149952, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 20561166336, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 20565622784, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 20575059968, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.24.attn_v.weight", "offset": 20584497152, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.25.ffn_gate.0.weight", "offset": 20588953600, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.25.ffn_down.0.weight", "offset": 20621983744, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_up.0.weight", "offset": 20655013888, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_gate.1.weight", "offset": 20688044032, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_down.1.weight", "offset": 20721074176, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_up.1.weight", "offset": 20754104320, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_gate.2.weight", "offset": 20787134464, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_down.2.weight", "offset": 20820164608, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_up.2.weight", "offset": 20853194752, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_gate.3.weight", "offset": 20886224896, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_down.3.weight", "offset": 20919255040, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_up.3.weight", "offset": 20952285184, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 20985315328, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.25.attn_k.weight", "offset": 20985380864, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 20989837312, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 20999274496, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.25.attn_v.weight", "offset": 21008711680, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.25.ffn_gate.4.weight", "offset": 21013168128, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.25.ffn_down.4.weight", "offset": 21046198272, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_up.4.weight", "offset": 21079228416, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_gate.5.weight", "offset": 21112258560, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_down.5.weight", "offset": 21145288704, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_up.5.weight", "offset": 21178318848, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_gate.6.weight", "offset": 21211348992, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_down.6.weight", "offset": 21244379136, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_up.6.weight", "offset": 21277409280, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_gate.7.weight", "offset": 21310439424, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_down.7.weight", "offset": 21343469568, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.25.ffn_up.7.weight", "offset": 21376499712, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 21409529856, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 21409546240, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.ffn_gate.0.weight", "offset": 21409562624, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.26.ffn_down.0.weight", "offset": 21442592768, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_up.0.weight", "offset": 21475622912, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_gate.1.weight", "offset": 21508653056, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_down.1.weight", "offset": 21541683200, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_up.1.weight", "offset": 21574713344, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_gate.2.weight", "offset": 21607743488, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_down.2.weight", "offset": 21640773632, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_up.2.weight", "offset": 21673803776, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_gate.3.weight", "offset": 21706833920, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_down.3.weight", "offset": 21739864064, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_up.3.weight", "offset": 21772894208, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_gate.4.weight", "offset": 21805924352, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_down.4.weight", "offset": 21838954496, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_up.4.weight", "offset": 21871984640, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_gate.5.weight", "offset": 21905014784, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_down.5.weight", "offset": 21938044928, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_up.5.weight", "offset": 21971075072, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_gate.6.weight", "offset": 22004105216, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_down.6.weight", "offset": 22037135360, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_up.6.weight", "offset": 22070165504, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_gate.7.weight", "offset": 22103195648, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_down.7.weight", "offset": 22136225792, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_up.7.weight", "offset": 22169255936, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 22202286080, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 22202351616, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 22202368000, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 22202384384, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 22206840832, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 22216278016, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.26.attn_v.weight", "offset": 22225715200, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.27.ffn_gate.0.weight", "offset": 22230171648, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.27.ffn_down.0.weight", "offset": 22263201792, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_up.0.weight", "offset": 22296231936, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_gate.1.weight", "offset": 22329262080, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_down.1.weight", "offset": 22362292224, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 22395322368, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.27.attn_k.weight", "offset": 22395387904, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 22399844352, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.27.attn_q.weight", "offset": 22409281536, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.27.attn_v.weight", "offset": 22418718720, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.27.ffn_up.1.weight", "offset": 22423175168, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.27.ffn_gate.2.weight", "offset": 22456205312, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_down.2.weight", "offset": 22489235456, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_up.2.weight", "offset": 22522265600, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_gate.3.weight", "offset": 22555295744, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_down.3.weight", "offset": 22588325888, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_up.3.weight", "offset": 22621356032, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_gate.4.weight", "offset": 22654386176, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_down.4.weight", "offset": 22687416320, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_up.4.weight", "offset": 22720446464, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "\u0016\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.31.ffn_norm.", "offset": 22753476608, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_down.5.weight", "offset": 22786506752, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_up.5.weight", "offset": 22819536896, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_gate.6.weight", "offset": 22852567040, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_down.6.weight", "offset": 22885597184, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_up.6.weight", "offset": 22918627328, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_gate.7.weight", "offset": 22951657472, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_down.7.weight", "offset": 22984687616, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.27.ffn_up.7.weight", "offset": 23017717760, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 23050747904, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 23050764288, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.28.ffn_gate.0.weight", "offset": 23050780672, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.28.ffn_down.0.weight", "offset": 23083810816, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_up.0.weight", "offset": 23116840960, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_gate.1.weight", "offset": 23149871104, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_down.1.weight", "offset": 23182901248, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_up.1.weight", "offset": 23215931392, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_gate.2.weight", "offset": 23248961536, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_down.2.weight", "offset": 23281991680, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_up.2.weight", "offset": 23315021824, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_gate.3.weight", "offset": 23348051968, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_down.3.weight", "offset": 23381082112, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_up.3.weight", "offset": 23414112256, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_gate.4.weight", "offset": 23447142400, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_down.4.weight", "offset": 23480172544, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_up.4.weight", "offset": 23513202688, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_gate.5.weight", "offset": 23546232832, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_down.5.weight", "offset": 23579262976, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_up.5.weight", "offset": 23612293120, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_gate.6.weight", "offset": 23645323264, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_down.6.weight", "offset": 23678353408, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_up.6.weight", "offset": 23711383552, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_gate.7.weight", "offset": 23744413696, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 23777443840, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.28.attn_k.weight", "offset": 23777509376, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 23781965824, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.28.attn_q.weight", "offset": 23791403008, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.28.attn_v.weight", "offset": 23800840192, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.28.ffn_down.7.weight", "offset": 23805296640, "shape": [ 14336, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.28.ffn_up.7.weight", "offset": 23838326784, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 23871356928, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 23871373312, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.ffn_gate.0.weight", "offset": 23871389696, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.29.ffn_down.0.weight", "offset": 23904419840, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_up.0.weight", "offset": 23937449984, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_gate.1.weight", "offset": 23970480128, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_down.1.weight", "offset": 24003510272, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_up.1.weight", "offset": 24036540416, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_gate.2.weight", "offset": 24069570560, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_down.2.weight", "offset": 24102600704, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_up.2.weight", "offset": 24135630848, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_gate.3.weight", "offset": 24168660992, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_down.3.weight", "offset": 24201691136, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_up.3.weight", "offset": 24234721280, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_gate.4.weight", "offset": 24267751424, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_down.4.weight", "offset": 24300781568, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_up.4.weight", "offset": 24333811712, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_gate.5.weight", "offset": 24366841856, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_down.5.weight", "offset": 24399872000, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_up.5.weight", "offset": 24432902144, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_gate.6.weight", "offset": 24465932288, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_down.6.weight", "offset": 24498962432, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_up.6.weight", "offset": 24531992576, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_gate.7.weight", "offset": 24565022720, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_down.7.weight", "offset": 24598052864, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_up.7.weight", "offset": 24631083008, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 24664113152, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.29.attn_norm.weight", "offset": 24664178688, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 24664195072, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 24664211456, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 24668667904, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.29.attn_q.weight", "offset": 24678105088, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.29.attn_v.weight", "offset": 24687542272, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "blk.30.ffn_gate.0.weight", "offset": 24691998720, "shape": [ 4096, 14336 ], "size": 4456448, "type": 12 }, { "name": "blk.30.ffn_down.0.weight", "offset": 24725028864, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_up.0.weight", "offset": 24758059008, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_gate.1.weight", "offset": 24791089152, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_down.1.weight", "offset": 24824119296, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_up.1.weight", "offset": 24857149440, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_gate.2.weight", "offset": 24890179584, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_down.2.weight", "offset": 24923209728, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_up.2.weight", "offset": 24956239872, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_gate.3.weight", "offset": 24989270016, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_down.3.weight", "offset": 25022300160, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_up.3.weight", "offset": 25055330304, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_gate.4.weight", "offset": 25088360448, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_down.4.weight", "offset": 25121390592, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_up.4.weight", "offset": 25154420736, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 25187450880, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.30.attn_k.weight", "offset": 25187516416, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 25191972864, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.30.attn_q.weight", "offset": 25201410048, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.30.attn_v.weight", "offset": 25210847232, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "output.weight", "offset": 25215303680, "shape": [ 4096, 32000 ], "size": 4456448, "type": 14 }, { "name": "blk.30.ffn_gate.5.weight", "offset": 25322823680, "shape": [ 4096, 14336 ], "size": 107520000, "type": 12 }, { "name": "blk.30.ffn_down.5.weight", "offset": 25355853824, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_up.5.weight", "offset": 25388883968, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_gate.6.weight", "offset": 25421914112, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_down.6.weight", "offset": 25454944256, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_up.6.weight", "offset": 25487974400, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_gate.7.weight", "offset": 25521004544, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_down.7.weight", "offset": 25554034688, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.30.ffn_up.7.weight", "offset": 25587064832, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 25620094976, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 25620111360, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.ffn_gate.0.weight", "offset": 25620127744, "shape": [ 4096, 14336 ], "size": 16384, "type": 12 }, { "name": "blk.31.ffn_down.0.weight", "offset": 25653157888, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_up.0.weight", "offset": 25686188032, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_gate.1.weight", "offset": 25719218176, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_down.1.weight", "offset": 25752248320, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_up.1.weight", "offset": 25785278464, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_gate.2.weight", "offset": 25818308608, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_down.2.weight", "offset": 25851338752, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_up.2.weight", "offset": 25884368896, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_gate.3.weight", "offset": 25917399040, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_down.3.weight", "offset": 25950429184, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_up.3.weight", "offset": 25983459328, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_gate.4.weight", "offset": 26016489472, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_down.4.weight", "offset": 26049519616, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_up.4.weight", "offset": 26082549760, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_gate.5.weight", "offset": 26115579904, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_down.5.weight", "offset": 26148610048, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_up.5.weight", "offset": 26181640192, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_gate.6.weight", "offset": 26214670336, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_down.6.weight", "offset": 26247700480, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_up.6.weight", "offset": 26280730624, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_gate.7.weight", "offset": 26313760768, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_down.7.weight", "offset": 26346790912, "shape": [ 14336, 4096 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_up.7.weight", "offset": 26379821056, "shape": [ 4096, 14336 ], "size": 33030144, "type": 12 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 26412851200, "shape": [ 4096, 8 ], "size": 33030144, "type": 1 }, { "name": "blk.31.attn_norm.weight", "offset": 26412916736, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 26412933120, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 26412949504, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 26417405952, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.31.attn_q.weight", "offset": 26426843136, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.31.attn_v.weight", "offset": 26436280320, "shape": [ 4096, 1024 ], "size": 9437184, "type": 8 }, { "name": "output_norm.weight", "offset": 26440736768, "shape": [ 4096 ], "size": 4456448, "type": 0 } ], "version": 3 }