mixtral

{ "metadata": { "general.architecture": "llama", "general.file_type": 15, "general.name": "Mixtral-8x22B-Instruct-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 65536, "llama.embedding_length": 6144, "llama.expert_count": 8, "llama.expert_used_count": 2, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.chat_template.tool_use": "{{bos_token}}{% set user_messages = messages | selectattr('role', 'equalto', 'user') | list %}{% for message in messages %}{% if message['role'] == 'user' %}{% if message == user_messages[-1] %}{% if tools %}{{'[AVAILABLE_TOOLS]'+ tools|string + '[/AVAILABLE_TOOLS]'}}{% endif %}{{ '[INST]' + message['content'] + '[/INST]' }}{% else %}{{ '[INST]' + message['content'] + '[/INST]' }}{% endif %}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + ' ' + eos_token}}{% elif message['role'] == 'tool_results' %}{{'[TOOL_RESULTS]' + message['content']|string + '[/TOOL_RESULTS]'}}{% elif message['role'] == 'tool_calls' %}{{'[TOOL_CALLS]' + message['content']|string + eos_token}}{% endif %}{% endfor %}", "tokenizer.chat_templates": "tool_use", "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 140630071296, "tensors": [ { "name": "blk.0.ffn_gate_inp.weight", "offset": 113246208, "shape": [ 6144, 8 ], "size": 113246208, "type": 1 }, { "name": "blk.0.attn_k.weight", "offset": 113344512, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 120029184, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 145981440, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.0.attn_v.weight", "offset": 167215104, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.0.ffn_gate_exps.weight", "offset": 173899776, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.0.ffn_down_exps.weight", "offset": 626884608, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.0.ffn_up_exps.weight", "offset": 1287487488, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.0.attn_norm.weight", "offset": 1740472320, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 1740496896, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1740521472, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.1.attn_k.weight", "offset": 1740619776, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 1747304448, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 1773256704, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.1.attn_v.weight", "offset": 1794490368, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 1801175040, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 2254159872, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 2914762752, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.1.attn_norm.weight", "offset": 3367747584, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 3367772160, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 3367796736, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.2.attn_k.weight", "offset": 3367895040, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 3374579712, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 3400531968, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.2.attn_v.weight", "offset": 3421765632, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 3428450304, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 3881435136, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 4542038016, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 4995022848, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 4995047424, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 4995072000, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.3.attn_k.weight", "offset": 4995170304, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 5001854976, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 5027807232, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.3.attn_v.weight", "offset": 5049040896, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 5055725568, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 5508710400, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 6169313280, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 6622298112, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 6622322688, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 6622347264, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.4.attn_k.weight", "offset": 6622445568, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 6629130240, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 6655082496, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.4.attn_v.weight", "offset": 6676316160, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 6683000832, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 7135985664, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 7796588544, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 8249573376, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 8249597952, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 8249622528, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.5.attn_k.weight", "offset": 8249720832, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 8256405504, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 8282357760, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.5.attn_v.weight", "offset": 8303591424, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 8310276096, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 8763260928, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 9423863808, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 9876848640, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 9876873216, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 9876897792, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.6.attn_k.weight", "offset": 9876996096, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 9883680768, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 9909633024, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.6.attn_v.weight", "offset": 9930866688, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 9937551360, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 10390536192, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 11051139072, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 11504123904, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 11504148480, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 11504173056, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.7.attn_k.weight", "offset": 11504271360, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 11510956032, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 11536908288, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.7.attn_v.weight", "offset": 11558141952, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 11564826624, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 12017811456, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 12470796288, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 12923781120, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 12923805696, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 12923830272, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.8.attn_k.weight", "offset": 12923928576, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 12930613248, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 12956565504, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.8.attn_v.weight", "offset": 12977799168, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 12984483840, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 13437468672, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 13890453504, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 14343438336, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 14343462912, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 14343487488, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.9.attn_k.weight", "offset": 14343585792, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 14350270464, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 14376222720, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.9.attn_v.weight", "offset": 14397456384, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 14404141056, "shape": [ 6144, 8 ], "size": 6684672, "type": 1 }, { "name": "blk.10.attn_k.weight", "offset": 14404239360, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 14410924032, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 14436876288, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.10.attn_v.weight", "offset": 14458109952, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 14464794624, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 14917779456, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 15578382336, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 16031367168, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 16031391744, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 16031416320, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 12 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 16484401152, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 16937385984, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 17390370816, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 17390395392, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 17390419968, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.11.attn_k.weight", "offset": 17390518272, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 17397202944, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 17423155200, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.11.attn_v.weight", "offset": 17444388864, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 17451073536, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 17904058368, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 18357043200, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 18810028032, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 18810052608, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 18810077184, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.12.attn_k.weight", "offset": 18810175488, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 18816860160, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 18842812416, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.12.attn_v.weight", "offset": 18864046080, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 18870730752, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 19323715584, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 19984318464, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 20437303296, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 20437327872, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 20437352448, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.13.attn_k.weight", "offset": 20437450752, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 20444135424, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 20470087680, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.13.attn_v.weight", "offset": 20491321344, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 20498006016, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 20950990848, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 21403975680, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 21856960512, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 21856985088, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 21857009664, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.14.attn_k.weight", "offset": 21857107968, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 21863792640, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 21889744896, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.14.attn_v.weight", "offset": 21910978560, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 21917663232, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 22370648064, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 22823632896, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 23276617728, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 23276642304, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 23276666880, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.15.attn_k.weight", "offset": 23276765184, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 23283449856, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 23309402112, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.15.attn_v.weight", "offset": 23330635776, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 23337320448, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 23790305280, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 24450908160, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 24903892992, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 24903917568, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 24903942144, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.16.attn_k.weight", "offset": 24904040448, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 24910725120, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 24936677376, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.16.attn_v.weight", "offset": 24957911040, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 24964595712, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 25417580544, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 25870565376, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 26323550208, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 26323574784, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 26323599360, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.17.attn_k.weight", "offset": 26323697664, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 26330382336, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 26356334592, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.17.attn_v.weight", "offset": 26377568256, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 26384252928, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 26837237760, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 27290222592, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 27743207424, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 27743232000, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 27743256576, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.18.attn_k.weight", "offset": 27743354880, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 27750039552, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 27775991808, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.18.attn_v.weight", "offset": 27797225472, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 27803910144, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 28256894976, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 28917497856, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 29370482688, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 29370507264, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 29370531840, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.19.attn_k.weight", "offset": 29370630144, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 29377314816, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 29403267072, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.19.attn_v.weight", "offset": 29424500736, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 29431185408, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 29884170240, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 30337155072, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 30790139904, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 30790164480, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 30790189056, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.20.attn_k.weight", "offset": 30790287360, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 30796972032, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 30822924288, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.20.attn_v.weight", "offset": 30844157952, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 30850842624, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 31303827456, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 31756812288, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 32209797120, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 32209821696, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 32209846272, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.21.attn_k.weight", "offset": 32209944576, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 32216629248, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 32242581504, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.21.attn_v.weight", "offset": 32263815168, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 32270499840, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 32723484672, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 33384087552, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 33837072384, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 33837096960, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 33837121536, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.22.attn_q.weight", "offset": 33843806208, "shape": [ 6144, 6144 ], "size": 6684672, "type": 12 }, { "name": "blk.22.attn_v.weight", "offset": 33865039872, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 33871724544, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 34324709376, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 34777694208, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 35230679040, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 35230777344, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 35230801920, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 35230826496, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.23.attn_k.weight", "offset": 35256778752, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.23.attn_q.weight", "offset": 35263463424, "shape": [ 6144, 6144 ], "size": 6684672, "type": 12 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 35284697088, "shape": [ 6144, 16384, 8 ], "size": 21233664, "type": 12 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 35737681920, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 36190666752, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 36643651584, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 36643749888, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 36643774464, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 36643799040, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.23.attn_v.weight", "offset": 36669751296, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.24.attn_q.weight", "offset": 36676435968, "shape": [ 6144, 6144 ], "size": 6684672, "type": 12 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 36697669632, "shape": [ 6144, 16384, 8 ], "size": 21233664, "type": 12 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 37150654464, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 37811257344, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 38264242176, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 38264340480, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 38264365056, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 38264389632, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 38271074304, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.24.attn_v.weight", "offset": 38297026560, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 38303711232, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 38756696064, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 39209680896, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.25.attn_k.weight", "offset": 39209779200, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 39216463872, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 39242416128, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.25.attn_v.weight", "offset": 39263649792, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 39270334464, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 39723319296, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 39723343872, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 39723368448, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 12 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 40176353280, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.26.attn_k.weight", "offset": 40176451584, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 40183136256, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 40209088512, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.26.attn_v.weight", "offset": 40230322176, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 40237006848, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 40689991680, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 41142976512, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 41143001088, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 41143025664, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.27.attn_k.weight", "offset": 41143123968, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 41149808640, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.27.attn_q.weight", "offset": 41175760896, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.27.attn_v.weight", "offset": 41196994560, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 41203679232, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 41656664064, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 42317266944, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 42770251776, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 42770276352, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 42770300928, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.28.attn_k.weight", "offset": 42770399232, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 42777083904, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 42803036160, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.28.attn_v.weight", "offset": 42824269824, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 42830954496, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 43283939328, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 43736924160, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 44189908992, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 44189933568, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 44189958144, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.29.attn_k.weight", "offset": 44190056448, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 44196741120, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 44222693376, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.29.attn_v.weight", "offset": 44243927040, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 44250611712, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 44703596544, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 45156581376, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 45609566208, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 45609590784, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 45609615360, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.30.attn_k.weight", "offset": 45609713664, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 45616398336, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 45642350592, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.30.attn_v.weight", "offset": 45663584256, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 45670268928, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 46123253760, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 46783856640, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 47236841472, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 47236866048, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 47236890624, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.31.attn_k.weight", "offset": 47236988928, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 47243673600, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.31.attn_q.weight", "offset": 47269625856, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.31.attn_v.weight", "offset": 47290859520, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 47297544192, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 47750529024, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 48203513856, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 48656498688, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 48656523264, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 48656547840, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.32.attn_k.weight", "offset": 48656646144, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.32.attn_output.weight", "offset": 48663330816, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.32.attn_q.weight", "offset": 48689283072, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.32.attn_v.weight", "offset": 48710516736, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 48717201408, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 49170186240, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 49623171072, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 50076155904, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 50076180480, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 50076205056, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.33.attn_k.weight", "offset": 50076303360, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.33.attn_output.weight", "offset": 50082988032, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.33.attn_q.weight", "offset": 50108940288, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.33.attn_v.weight", "offset": 50130173952, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 50136858624, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 50589843456, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 51250446336, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 51703431168, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 51703455744, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 51703480320, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.34.attn_k.weight", "offset": 51703578624, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.34.attn_output.weight", "offset": 51710263296, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.34.attn_q.weight", "offset": 51736215552, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.34.attn_v.weight", "offset": 51757449216, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 51764133888, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 52217118720, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 52670103552, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 53123088384, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 53123112960, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 53123137536, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.35.attn_k.weight", "offset": 53123235840, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.35.attn_output.weight", "offset": 53129920512, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.35.attn_q.weight", "offset": 53155872768, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.35.attn_v.weight", "offset": 53177106432, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 53183791104, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 53636775936, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 54089760768, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.35.attn_norm.weight", "offset": 54542745600, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 54542770176, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 54542794752, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.36.attn_k.weight", "offset": 54542893056, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.36.attn_output.weight", "offset": 54549577728, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.36.attn_q.weight", "offset": 54575529984, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.36.attn_v.weight", "offset": 54596763648, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 54603448320, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 55056433152, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 55717036032, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.36.attn_norm.weight", "offset": 56170020864, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 56170045440, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 56170070016, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.37.attn_k.weight", "offset": 56170168320, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.37.attn_output.weight", "offset": 56176852992, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.37.attn_q.weight", "offset": 56202805248, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.37.attn_v.weight", "offset": 56224038912, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 56230723584, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 56683708416, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 57136693248, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 57589678080, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 57589702656, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 57589727232, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.38.attn_k.weight", "offset": 57589825536, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.38.attn_output.weight", "offset": 57596510208, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.38.attn_q.weight", "offset": 57622462464, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.38.attn_v.weight", "offset": 57643696128, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 57650380800, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 58103365632, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 58556350464, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 59009335296, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 59009359872, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 59009384448, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.39.attn_k.weight", "offset": 59009482752, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.39.attn_output.weight", "offset": 59016167424, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.39.attn_q.weight", "offset": 59042119680, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.39.attn_v.weight", "offset": 59063353344, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 59070038016, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 59523022848, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 60183625728, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 60636610560, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 60636635136, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 60636659712, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.40.attn_k.weight", "offset": 60636758016, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.40.attn_output.weight", "offset": 60643442688, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.40.attn_q.weight", "offset": 60669394944, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.40.attn_v.weight", "offset": 60690628608, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 60697313280, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 61150298112, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 61603282944, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.40.attn_norm.weight", "offset": 62056267776, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 62056292352, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 62056316928, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.41.attn_k.weight", "offset": 62056415232, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.41.attn_output.weight", "offset": 62063099904, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.41.attn_q.weight", "offset": 62089052160, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.41.attn_v.weight", "offset": 62110285824, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 62116970496, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 62569955328, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 63022940160, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.41.attn_norm.weight", "offset": 63475924992, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 63475949568, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 63475974144, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.42.attn_k.weight", "offset": 63476072448, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.42.attn_output.weight", "offset": 63482757120, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.42.attn_q.weight", "offset": 63508709376, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.42.attn_v.weight", "offset": 63529943040, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 63536627712, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 63989612544, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 64650215424, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.42.attn_norm.weight", "offset": 65103200256, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 65103224832, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 65103249408, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.43.attn_k.weight", "offset": 65103347712, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.43.attn_output.weight", "offset": 65110032384, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.43.attn_q.weight", "offset": 65135984640, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.43.attn_v.weight", "offset": 65157218304, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 65163902976, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 65616887808, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 66069872640, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.43.attn_norm.weight", "offset": 66522857472, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 66522882048, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 66522906624, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.44.attn_k.weight", "offset": 66523004928, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.44.attn_output.weight", "offset": 66529689600, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.44.attn_q.weight", "offset": 66555641856, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.44.attn_v.weight", "offset": 66576875520, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 66583560192, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 67036545024, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 67489529856, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.44.attn_norm.weight", "offset": 67942514688, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 67942539264, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 67942563840, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.45.attn_k.weight", "offset": 67942662144, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.45.attn_output.weight", "offset": 67949346816, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.45.attn_q.weight", "offset": 67975299072, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.45.attn_v.weight", "offset": 67996532736, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 68003217408, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 68456202240, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 69116805120, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.45.attn_norm.weight", "offset": 69569789952, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 69569814528, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 69569839104, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.46.attn_k.weight", "offset": 69569937408, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.46.attn_output.weight", "offset": 69576622080, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.46.attn_q.weight", "offset": 69602574336, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.46.attn_v.weight", "offset": 69623808000, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 69630492672, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 70083477504, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 70536462336, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.46.attn_norm.weight", "offset": 70989447168, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.46.ffn_norm.weight", "offset": 70989471744, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 70989496320, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.47.attn_k.weight", "offset": 70989594624, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.47.attn_output.weight", "offset": 70996279296, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.47.attn_q.weight", "offset": 71022231552, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.47.attn_v.weight", "offset": 71043465216, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 71050149888, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 71503134720, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 71956119552, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 12 }, { "name": "blk.47.attn_norm.weight", "offset": 72409104384, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.47.ffn_norm.weight", "offset": 72409128960, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 72409153536, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.48.attn_k.weight", "offset": 72409251840, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.48.attn_output.weight", "offset": 72415936512, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.48.attn_q.weight", "offset": 72441888768, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.48.attn_v.weight", "offset": 72463122432, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 72469807104, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 72922791936, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 73583394816, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.48.attn_norm.weight", "offset": 74036379648, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.48.ffn_norm.weight", "offset": 74036404224, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 74036428800, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.49.attn_q.weight", "offset": 74043113472, "shape": [ 6144, 6144 ], "size": 6684672, "type": 12 }, { "name": "blk.49.attn_v.weight", "offset": 74064347136, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 74071031808, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 74524016640, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 75184619520, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 75637604352, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.49.attn_norm.weight", "offset": 75637702656, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.49.ffn_norm.weight", "offset": 75637727232, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.49.attn_output.weight", "offset": 75637751808, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.50.attn_k.weight", "offset": 75663704064, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.50.attn_q.weight", "offset": 75670388736, "shape": [ 6144, 6144 ], "size": 6684672, "type": 12 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 75691622400, "shape": [ 6144, 16384, 8 ], "size": 21233664, "type": 12 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 76144607232, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 76805210112, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 77258194944, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.50.attn_norm.weight", "offset": 77258293248, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.50.ffn_norm.weight", "offset": 77258317824, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 77258342400, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.50.attn_v.weight", "offset": 77284294656, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.51.attn_q.weight", "offset": 77290979328, "shape": [ 6144, 6144 ], "size": 6684672, "type": 12 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 77312212992, "shape": [ 6144, 16384, 8 ], "size": 21233664, "type": 12 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 77765197824, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 78425800704, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 78878785536, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.51.attn_norm.weight", "offset": 78878883840, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.51.ffn_norm.weight", "offset": 78878908416, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 78878932992, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.51.attn_output.weight", "offset": 78885617664, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.51.attn_v.weight", "offset": 78911569920, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 78918254592, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 79371239424, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 80031842304, "shape": [ 6144, 8 ], "size": 660602880, "type": 1 }, { "name": "blk.52.attn_k.weight", "offset": 80031940608, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.52.attn_output.weight", "offset": 80038625280, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.52.attn_q.weight", "offset": 80064577536, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.52.attn_v.weight", "offset": 80085811200, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 80092495872, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.52.attn_norm.weight", "offset": 80545480704, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.52.ffn_norm.weight", "offset": 80545505280, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 80545529856, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 12 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 80998514688, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.53.attn_k.weight", "offset": 80998612992, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.53.attn_output.weight", "offset": 81005297664, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.53.attn_q.weight", "offset": 81031249920, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.53.attn_v.weight", "offset": 81052483584, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 81059168256, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 14 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 81719771136, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.53.attn_norm.weight", "offset": 82172755968, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.53.ffn_norm.weight", "offset": 82172780544, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 82172805120, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.54.attn_k.weight", "offset": 82172903424, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.54.attn_output.weight", "offset": 82179588096, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.54.attn_q.weight", "offset": 82205540352, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.54.attn_v.weight", "offset": 82226774016, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 82233458688, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 12 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 82686443520, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 83347046400, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.54.attn_norm.weight", "offset": 83800031232, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.54.ffn_norm.weight", "offset": 83800055808, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 83800080384, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.55.attn_k.weight", "offset": 83800178688, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.55.attn_output.weight", "offset": 83806863360, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.55.attn_q.weight", "offset": 83832815616, "shape": [ 6144, 6144 ], "size": 25952256, "type": 12 }, { "name": "blk.55.attn_v.weight", "offset": 83854049280, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "output.weight", "offset": 83860733952, "shape": [ 6144, 32768 ], "size": 6684672, "type": 14 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 84025884672, "shape": [ 6144, 16384, 8 ], "size": 165150720, "type": 12 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 84478869504, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 14 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 85139472384, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 12 }, { "name": "blk.55.attn_norm.weight", "offset": 85592457216, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.55.ffn_norm.weight", "offset": 85592481792, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "output_norm.weight", "offset": 85592506368, "shape": [ 6144 ], "size": 24576, "type": 0 } ], "version": 3 }