mixtral

{ "metadata": { "general.architecture": "llama", "general.file_type": 2, "general.name": "Mixtral-8x22B-Instruct-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 65536, "llama.embedding_length": 6144, "llama.expert_count": 8, "llama.expert_used_count": 2, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.chat_template.tool_use": "{{bos_token}}{% set user_messages = messages | selectattr('role', 'equalto', 'user') | list %}{% for message in messages %}{% if message['role'] == 'user' %}{% if message == user_messages[-1] %}{% if tools %}{{'[AVAILABLE_TOOLS]'+ tools|string + '[/AVAILABLE_TOOLS]'}}{% endif %}{{ '[INST]' + message['content'] + '[/INST]' }}{% else %}{{ '[INST]' + message['content'] + '[/INST]' }}{% endif %}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + ' ' + eos_token}}{% elif message['role'] == 'tool_results' %}{{'[TOOL_RESULTS]' + message['content']|string + '[/TOOL_RESULTS]'}}{% elif message['role'] == 'tool_calls' %}{{'[TOOL_CALLS]' + message['content']|string + eos_token}}{% endif %}{% endfor %}", "tokenizer.chat_templates": "tool_use", "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 140630071296, "tensors": [ { "name": "blk.0.ffn_gate_inp.weight", "offset": 113246208, "shape": [ 6144, 8 ], "size": 113246208, "type": 1 }, { "name": "blk.0.attn_k.weight", "offset": 113344512, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 120029184, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.0.attn_q.weight", "offset": 141262848, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.0.attn_v.weight", "offset": 162496512, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.0.ffn_gate_exps.weight", "offset": 169181184, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.0.ffn_down_exps.weight", "offset": 622166016, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.0.ffn_up_exps.weight", "offset": 1075150848, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.0.attn_norm.weight", "offset": 1528135680, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 1528160256, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1528184832, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.1.attn_k.weight", "offset": 1528283136, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 1534967808, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.1.attn_q.weight", "offset": 1556201472, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.1.attn_v.weight", "offset": 1577435136, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 1584119808, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 2037104640, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 2490089472, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 2943074304, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 2943098880, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 2943123456, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.2.attn_k.weight", "offset": 2943221760, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 2949906432, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.2.attn_q.weight", "offset": 2971140096, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.2.attn_v.weight", "offset": 2992373760, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 2999058432, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 3452043264, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 3905028096, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 4358012928, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 4358037504, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 4358062080, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.3.attn_k.weight", "offset": 4358160384, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 4364845056, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.3.attn_q.weight", "offset": 4386078720, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.3.attn_v.weight", "offset": 4407312384, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 4413997056, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 4866981888, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 5319966720, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 5772951552, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 5772976128, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 5773000704, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.4.attn_k.weight", "offset": 5773099008, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 5779783680, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.4.attn_q.weight", "offset": 5801017344, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.4.attn_v.weight", "offset": 5822251008, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 5828935680, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 6281920512, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 6734905344, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 7187890176, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 7187914752, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 7187939328, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.5.attn_k.weight", "offset": 7188037632, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 7194722304, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.5.attn_q.weight", "offset": 7215955968, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.5.attn_v.weight", "offset": 7237189632, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 7243874304, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 7696859136, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 8149843968, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 8602828800, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 8602853376, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 8602877952, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.6.attn_k.weight", "offset": 8602976256, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 8609660928, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.6.attn_q.weight", "offset": 8630894592, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.6.attn_v.weight", "offset": 8652128256, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 8658812928, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 9111797760, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 9564782592, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 10017767424, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 10017792000, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 10017816576, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.7.attn_k.weight", "offset": 10017914880, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 10024599552, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.7.attn_q.weight", "offset": 10045833216, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.7.attn_v.weight", "offset": 10067066880, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 10073751552, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 10526736384, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 10979721216, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.7.attn_norm.weight", "offset": 11432706048, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 11432730624, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 11432755200, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.8.attn_k.weight", "offset": 11432853504, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 11439538176, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.8.attn_q.weight", "offset": 11460771840, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.8.attn_v.weight", "offset": 11482005504, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 11488690176, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 11941675008, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 12394659840, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 12847644672, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 12847669248, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 12847693824, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.9.attn_k.weight", "offset": 12847792128, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 12854476800, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.9.attn_q.weight", "offset": 12875710464, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.9.attn_v.weight", "offset": 12896944128, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 12903628800, "shape": [ 6144, 8 ], "size": 6684672, "type": 1 }, { "name": "blk.10.attn_k.weight", "offset": 12903727104, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 12910411776, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.10.attn_q.weight", "offset": 12931645440, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.10.attn_v.weight", "offset": 12952879104, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 12959563776, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 13412548608, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 13865533440, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 14318518272, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 14318542848, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 14318567424, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 2 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 14771552256, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 15224537088, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 15677521920, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 15677546496, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 15677571072, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.11.attn_k.weight", "offset": 15677669376, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 15684354048, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.11.attn_q.weight", "offset": 15705587712, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.11.attn_v.weight", "offset": 15726821376, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 15733506048, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 16186490880, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 16639475712, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 17092460544, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 17092485120, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 17092509696, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.12.attn_k.weight", "offset": 17092608000, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 17099292672, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.12.attn_q.weight", "offset": 17120526336, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.12.attn_v.weight", "offset": 17141760000, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 17148444672, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 17601429504, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 18054414336, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 18507399168, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 18507423744, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 18507448320, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.13.attn_k.weight", "offset": 18507546624, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 18514231296, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.13.attn_q.weight", "offset": 18535464960, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.13.attn_v.weight", "offset": 18556698624, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 18563383296, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 19016368128, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 19469352960, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 19922337792, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 19922362368, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 19922386944, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.14.attn_k.weight", "offset": 19922485248, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 19929169920, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.14.attn_q.weight", "offset": 19950403584, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.14.attn_v.weight", "offset": 19971637248, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 19978321920, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 20431306752, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 20884291584, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 21337276416, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 21337300992, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 21337325568, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.15.attn_k.weight", "offset": 21337423872, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 21344108544, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.15.attn_q.weight", "offset": 21365342208, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.15.attn_v.weight", "offset": 21386575872, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 21393260544, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 21846245376, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 22299230208, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 22752215040, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 22752239616, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 22752264192, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.16.attn_k.weight", "offset": 22752362496, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 22759047168, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.16.attn_q.weight", "offset": 22780280832, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.16.attn_v.weight", "offset": 22801514496, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 22808199168, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 23261184000, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 23714168832, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 24167153664, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 24167178240, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 24167202816, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.17.attn_k.weight", "offset": 24167301120, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 24173985792, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.17.attn_q.weight", "offset": 24195219456, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.17.attn_v.weight", "offset": 24216453120, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 24223137792, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 24676122624, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 25129107456, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 25582092288, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 25582116864, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 25582141440, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.18.attn_k.weight", "offset": 25582239744, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 25588924416, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.18.attn_q.weight", "offset": 25610158080, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.18.attn_v.weight", "offset": 25631391744, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 25638076416, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 26091061248, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 26544046080, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 26997030912, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 26997055488, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 26997080064, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.19.attn_k.weight", "offset": 26997178368, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 27003863040, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.19.attn_q.weight", "offset": 27025096704, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.19.attn_v.weight", "offset": 27046330368, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 27053015040, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 27505999872, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 27958984704, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.19.attn_norm.weight", "offset": 28411969536, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 28411994112, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 28412018688, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.20.attn_k.weight", "offset": 28412116992, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 28418801664, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.20.attn_q.weight", "offset": 28440035328, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.20.attn_v.weight", "offset": 28461268992, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 28467953664, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 28920938496, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 29373923328, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 29826908160, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 29826932736, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 29826957312, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.21.attn_k.weight", "offset": 29827055616, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 29833740288, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.21.attn_q.weight", "offset": 29854973952, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.21.attn_v.weight", "offset": 29876207616, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 29882892288, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 30335877120, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 30788861952, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 31241846784, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 31241871360, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 31241895936, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.22.attn_q.weight", "offset": 31248580608, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.22.attn_v.weight", "offset": 31269814272, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 31276498944, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 31729483776, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 32182468608, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 32635453440, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 32635551744, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 32635576320, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 32635600896, "shape": [ 6144, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.23.attn_k.weight", "offset": 32656834560, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.23.attn_q.weight", "offset": 32663519232, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 32684752896, "shape": [ 6144, 16384, 8 ], "size": 21233664, "type": 2 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 33137737728, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 33590722560, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 34043707392, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 34043805696, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 34043830272, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 34043854848, "shape": [ 6144, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.23.attn_v.weight", "offset": 34065088512, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.24.attn_q.weight", "offset": 34071773184, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 34093006848, "shape": [ 6144, 16384, 8 ], "size": 21233664, "type": 2 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 34545991680, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 34998976512, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 35451961344, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 35452059648, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 35452084224, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 35452108800, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 35458793472, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.24.attn_v.weight", "offset": 35480027136, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 35486711808, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 35939696640, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 36392681472, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.25.attn_k.weight", "offset": 36392779776, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 36399464448, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.25.attn_q.weight", "offset": 36420698112, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.25.attn_v.weight", "offset": 36441931776, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 36448616448, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.25.attn_norm.weight", "offset": 36901601280, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 36901625856, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 36901650432, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 2 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 37354635264, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.26.attn_k.weight", "offset": 37354733568, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 37361418240, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.26.attn_q.weight", "offset": 37382651904, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.26.attn_v.weight", "offset": 37403885568, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 37410570240, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 37863555072, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 38316539904, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 38316564480, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 38316589056, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.27.attn_k.weight", "offset": 38316687360, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 38323372032, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.27.attn_q.weight", "offset": 38344605696, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.27.attn_v.weight", "offset": 38365839360, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 38372524032, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 38825508864, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 39278493696, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 39731478528, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 39731503104, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 39731527680, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.28.attn_k.weight", "offset": 39731625984, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 39738310656, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.28.attn_q.weight", "offset": 39759544320, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.28.attn_v.weight", "offset": 39780777984, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 39787462656, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 40240447488, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 40693432320, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.28.attn_norm.weight", "offset": 41146417152, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 41146441728, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 41146466304, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.29.attn_k.weight", "offset": 41146564608, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 41153249280, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.29.attn_q.weight", "offset": 41174482944, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.29.attn_v.weight", "offset": 41195716608, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 41202401280, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 41655386112, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 42108370944, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.29.attn_norm.weight", "offset": 42561355776, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 42561380352, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 42561404928, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.30.attn_k.weight", "offset": 42561503232, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 42568187904, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.30.attn_q.weight", "offset": 42589421568, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.30.attn_v.weight", "offset": 42610655232, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 42617339904, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 43070324736, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 43523309568, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.30.attn_norm.weight", "offset": 43976294400, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 43976318976, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 43976343552, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.31.attn_k.weight", "offset": 43976441856, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 43983126528, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.31.attn_q.weight", "offset": 44004360192, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.31.attn_v.weight", "offset": 44025593856, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 44032278528, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 44485263360, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 44938248192, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.31.attn_norm.weight", "offset": 45391233024, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 45391257600, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 45391282176, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.32.attn_k.weight", "offset": 45391380480, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.32.attn_output.weight", "offset": 45398065152, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.32.attn_q.weight", "offset": 45419298816, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.32.attn_v.weight", "offset": 45440532480, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 45447217152, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 45900201984, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 46353186816, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.32.attn_norm.weight", "offset": 46806171648, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 46806196224, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 46806220800, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.33.attn_k.weight", "offset": 46806319104, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.33.attn_output.weight", "offset": 46813003776, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.33.attn_q.weight", "offset": 46834237440, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.33.attn_v.weight", "offset": 46855471104, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 46862155776, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 47315140608, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 47768125440, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.33.attn_norm.weight", "offset": 48221110272, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 48221134848, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 48221159424, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.34.attn_k.weight", "offset": 48221257728, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.34.attn_output.weight", "offset": 48227942400, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.34.attn_q.weight", "offset": 48249176064, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.34.attn_v.weight", "offset": 48270409728, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 48277094400, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 48730079232, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 49183064064, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.34.attn_norm.weight", "offset": 49636048896, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 49636073472, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 49636098048, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.35.attn_k.weight", "offset": 49636196352, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.35.attn_output.weight", "offset": 49642881024, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.35.attn_q.weight", "offset": 49664114688, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.35.attn_v.weight", "offset": 49685348352, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 49692033024, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 50145017856, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 50598002688, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.35.attn_norm.weight", "offset": 51050987520, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 51051012096, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 51051036672, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.36.attn_k.weight", "offset": 51051134976, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.36.attn_output.weight", "offset": 51057819648, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.36.attn_q.weight", "offset": 51079053312, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.36.attn_v.weight", "offset": 51100286976, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 51106971648, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 51559956480, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 52012941312, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.36.attn_norm.weight", "offset": 52465926144, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 52465950720, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 52465975296, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.37.attn_k.weight", "offset": 52466073600, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.37.attn_output.weight", "offset": 52472758272, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.37.attn_q.weight", "offset": 52493991936, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.37.attn_v.weight", "offset": 52515225600, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 52521910272, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 52974895104, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 53427879936, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.37.attn_norm.weight", "offset": 53880864768, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 53880889344, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 53880913920, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.38.attn_k.weight", "offset": 53881012224, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.38.attn_output.weight", "offset": 53887696896, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.38.attn_q.weight", "offset": 53908930560, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.38.attn_v.weight", "offset": 53930164224, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 53936848896, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 54389833728, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 54842818560, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.38.attn_norm.weight", "offset": 55295803392, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 55295827968, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 55295852544, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.39.attn_k.weight", "offset": 55295950848, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.39.attn_output.weight", "offset": 55302635520, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.39.attn_q.weight", "offset": 55323869184, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.39.attn_v.weight", "offset": 55345102848, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 55351787520, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 55804772352, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 56257757184, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.39.attn_norm.weight", "offset": 56710742016, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 56710766592, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 56710791168, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.40.attn_k.weight", "offset": 56710889472, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.40.attn_output.weight", "offset": 56717574144, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.40.attn_q.weight", "offset": 56738807808, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.40.attn_v.weight", "offset": 56760041472, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 56766726144, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 57219710976, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 57672695808, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.40.attn_norm.weight", "offset": 58125680640, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 58125705216, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 58125729792, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.41.attn_k.weight", "offset": 58125828096, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.41.attn_output.weight", "offset": 58132512768, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.41.attn_q.weight", "offset": 58153746432, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.41.attn_v.weight", "offset": 58174980096, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 58181664768, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 58634649600, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 59087634432, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.41.attn_norm.weight", "offset": 59540619264, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 59540643840, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 59540668416, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.42.attn_k.weight", "offset": 59540766720, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.42.attn_output.weight", "offset": 59547451392, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.42.attn_q.weight", "offset": 59568685056, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.42.attn_v.weight", "offset": 59589918720, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 59596603392, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 60049588224, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 60502573056, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.42.attn_norm.weight", "offset": 60955557888, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 60955582464, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 60955607040, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.43.attn_k.weight", "offset": 60955705344, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.43.attn_output.weight", "offset": 60962390016, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.43.attn_q.weight", "offset": 60983623680, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.43.attn_v.weight", "offset": 61004857344, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 61011542016, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 61464526848, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 61917511680, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.43.attn_norm.weight", "offset": 62370496512, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 62370521088, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 62370545664, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.44.attn_k.weight", "offset": 62370643968, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.44.attn_output.weight", "offset": 62377328640, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.44.attn_q.weight", "offset": 62398562304, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.44.attn_v.weight", "offset": 62419795968, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "\u0002v\ufffd\ufffd\ufffd`t\ufffd\ufffdVifg\ufffdh\ufffd\ufffdt\ufffd\ufffd\ufffd}\ufffdFVf{", "offset": 62426480640, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 62879465472, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 63332450304, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.44.attn_norm.weight", "offset": 63785435136, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 63785459712, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 63785484288, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.45.attn_k.weight", "offset": 63785582592, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.45.attn_output.weight", "offset": 63792267264, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.45.attn_q.weight", "offset": 63813500928, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.45.attn_v.weight", "offset": 63834734592, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 63841419264, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 64294404096, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 64747388928, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.45.attn_norm.weight", "offset": 65200373760, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 65200398336, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 65200422912, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.46.attn_k.weight", "offset": 65200521216, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.46.attn_output.weight", "offset": 65207205888, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.46.attn_q.weight", "offset": 65228439552, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.46.attn_v.weight", "offset": 65249673216, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 65256357888, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 65709342720, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 66162327552, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.46.attn_norm.weight", "offset": 66615312384, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.46.ffn_norm.weight", "offset": 66615336960, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 66615361536, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.47.attn_k.weight", "offset": 66615459840, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.47.attn_output.weight", "offset": 66622144512, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.47.attn_q.weight", "offset": 66643378176, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.47.attn_v.weight", "offset": 66664611840, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 66671296512, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 67124281344, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 67577266176, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.47.attn_norm.weight", "offset": 68030251008, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.47.ffn_norm.weight", "offset": 68030275584, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 68030300160, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.48.attn_k.weight", "offset": 68030398464, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.48.attn_output.weight", "offset": 68037083136, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.48.attn_q.weight", "offset": 68058316800, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.48.attn_v.weight", "offset": 68079550464, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 68086235136, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 68539219968, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 68992204800, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.48.attn_norm.weight", "offset": 69445189632, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.48.ffn_norm.weight", "offset": 69445214208, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 69445238784, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.49.attn_q.weight", "offset": 69451923456, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.49.attn_v.weight", "offset": 69473157120, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 69479841792, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 69932826624, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 70385811456, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 70838796288, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.49.attn_norm.weight", "offset": 70838894592, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.49.ffn_norm.weight", "offset": 70838919168, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.49.attn_output.weight", "offset": 70838943744, "shape": [ 6144, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.50.attn_k.weight", "offset": 70860177408, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.50.attn_q.weight", "offset": 70866862080, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 70888095744, "shape": [ 6144, 16384, 8 ], "size": 21233664, "type": 2 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 71341080576, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 71794065408, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 72247050240, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.50.attn_norm.weight", "offset": 72247148544, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.50.ffn_norm.weight", "offset": 72247173120, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 72247197696, "shape": [ 6144, 6144 ], "size": 24576, "type": 2 }, { "name": "blk.50.attn_v.weight", "offset": 72268431360, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.51.attn_q.weight", "offset": 72275116032, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 72296349696, "shape": [ 6144, 16384, 8 ], "size": 21233664, "type": 2 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 72749334528, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 73202319360, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 73655304192, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.51.attn_norm.weight", "offset": 73655402496, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.51.ffn_norm.weight", "offset": 73655427072, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 73655451648, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.51.attn_output.weight", "offset": 73662136320, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.51.attn_v.weight", "offset": 73683369984, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 73690054656, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 74143039488, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 74596024320, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.52.attn_k.weight", "offset": 74596122624, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.52.attn_output.weight", "offset": 74602807296, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.52.attn_q.weight", "offset": 74624040960, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.52.attn_v.weight", "offset": 74645274624, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 74651959296, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.52.attn_norm.weight", "offset": 75104944128, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.52.ffn_norm.weight", "offset": 75104968704, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 75104993280, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 2 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 75557978112, "shape": [ 6144, 8 ], "size": 452984832, "type": 1 }, { "name": "blk.53.attn_k.weight", "offset": 75558076416, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.53.attn_output.weight", "offset": 75564761088, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.53.attn_q.weight", "offset": 75585994752, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.53.attn_v.weight", "offset": 75607228416, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 75613913088, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 76066897920, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.53.attn_norm.weight", "offset": 76519882752, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.53.ffn_norm.weight", "offset": 76519907328, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 76519931904, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.54.attn_k.weight", "offset": 76520030208, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.54.attn_output.weight", "offset": 76526714880, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.54.attn_q.weight", "offset": 76547948544, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.54.attn_v.weight", "offset": 76569182208, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 76575866880, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 2 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 77028851712, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 77481836544, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.54.attn_norm.weight", "offset": 77934821376, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.54.ffn_norm.weight", "offset": 77934845952, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 77934870528, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.55.attn_k.weight", "offset": 77934968832, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.55.attn_output.weight", "offset": 77941653504, "shape": [ 6144, 6144 ], "size": 6684672, "type": 2 }, { "name": "blk.55.attn_q.weight", "offset": 77962887168, "shape": [ 6144, 6144 ], "size": 21233664, "type": 2 }, { "name": "blk.55.attn_v.weight", "offset": 77984120832, "shape": [ 6144, 1024 ], "size": 21233664, "type": 8 }, { "name": "output.weight", "offset": 77990805504, "shape": [ 6144, 32768 ], "size": 6684672, "type": 14 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 78155956224, "shape": [ 6144, 16384, 8 ], "size": 165150720, "type": 2 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 78608941056, "shape": [ 16384, 6144, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 79061925888, "shape": [ 6144, 16384, 8 ], "size": 452984832, "type": 2 }, { "name": "blk.55.attn_norm.weight", "offset": 79514910720, "shape": [ 6144 ], "size": 452984832, "type": 0 }, { "name": "blk.55.ffn_norm.weight", "offset": 79514935296, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "output_norm.weight", "offset": 79514959872, "shape": [ 6144 ], "size": 24576, "type": 0 } ], "version": 3 }