mixtral

{ "metadata": { "general.architecture": "llama", "general.file_type": 11, "general.name": "Mixtral-8x22B-Instruct-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 65536, "llama.embedding_length": 6144, "llama.expert_count": 8, "llama.expert_used_count": 2, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.chat_template.tool_use": "{{bos_token}}{% set user_messages = messages | selectattr('role', 'equalto', 'user') | list %}{% for message in messages %}{% if message['role'] == 'user' %}{% if message == user_messages[-1] %}{% if tools %}{{'[AVAILABLE_TOOLS]'+ tools|string + '[/AVAILABLE_TOOLS]'}}{% endif %}{{ '[INST]' + message['content'] + '[/INST]' }}{% else %}{{ '[INST]' + message['content'] + '[/INST]' }}{% endif %}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + ' ' + eos_token}}{% elif message['role'] == 'tool_results' %}{{'[TOOL_RESULTS]' + message['content']|string + '[/TOOL_RESULTS]'}}{% elif message['role'] == 'tool_calls' %}{{'[TOOL_CALLS]' + message['content']|string + eos_token}}{% endif %}{% endfor %}", "tokenizer.chat_templates": "tool_use", "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 140630071296, "tensors": [ { "name": "blk.0.ffn_gate_inp.weight", "offset": 86507520, "shape": [ 6144, 8 ], "size": 86507520, "type": 1 }, { "name": "blk.0.attn_k.weight", "offset": 86605824, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 93290496, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 119242752, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 135462912, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.0.ffn_gate_exps.weight", "offset": 142147584, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.0.ffn_down_exps.weight", "offset": 488177664, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.0.ffn_up_exps.weight", "offset": 834207744, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.0.attn_norm.weight", "offset": 1180237824, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 1180262400, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1180286976, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.1.attn_k.weight", "offset": 1180385280, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 1187069952, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 1213022208, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 1229242368, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 1235927040, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 1581957120, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 1927987200, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 2274017280, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 2274041856, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 2274066432, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.2.attn_k.weight", "offset": 2274164736, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 2280849408, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 2306801664, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.2.attn_v.weight", "offset": 2323021824, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 2329706496, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 2675736576, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 3021766656, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 3367796736, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 3367821312, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 3367845888, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.3.attn_k.weight", "offset": 3367944192, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 3374628864, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 3400581120, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.3.attn_v.weight", "offset": 3416801280, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 3423485952, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 3769516032, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 4115546112, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 4461576192, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 4461600768, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 4461625344, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.4.attn_k.weight", "offset": 4461723648, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 4468408320, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 4494360576, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 4510580736, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 4517265408, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 4863295488, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 5209325568, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 5555355648, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 5555380224, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 5555404800, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.5.attn_k.weight", "offset": 5555503104, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 5562187776, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 5588140032, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 5604360192, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 5611044864, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 5957074944, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 6303105024, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 6649135104, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 6649159680, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 6649184256, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.6.attn_k.weight", "offset": 6649282560, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 6655967232, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 6681919488, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 6698139648, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 6704824320, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 7050854400, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 7396884480, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 7742914560, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 7742939136, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 7742963712, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.7.attn_k.weight", "offset": 7743062016, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 7749746688, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 7775698944, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 7791919104, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 7798603776, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 8144633856, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 8490663936, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.7.attn_norm.weight", "offset": 8836694016, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 8836718592, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 8836743168, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.8.attn_k.weight", "offset": 8836841472, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 8843526144, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 8869478400, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 8885698560, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 8892383232, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 9238413312, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 9584443392, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 9930473472, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 9930498048, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 9930522624, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.9.attn_k.weight", "offset": 9930620928, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 9937305600, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 9963257856, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.9.attn_v.weight", "offset": 9979478016, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 9986162688, "shape": [ 6144, 8 ], "size": 6684672, "type": 1 }, { "name": "blk.10.attn_k.weight", "offset": 9986260992, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 9992945664, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 10018897920, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 10035118080, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 10041802752, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 10387832832, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 10733862912, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 11079892992, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 11079917568, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 11079942144, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 11 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 11425972224, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 11772002304, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.10.attn_norm.weight", "offset": 12118032384, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 12118056960, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 12118081536, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.11.attn_k.weight", "offset": 12118179840, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 12124864512, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 12150816768, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.11.attn_v.weight", "offset": 12167036928, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 12173721600, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 12519751680, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 12865781760, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 13211811840, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 13211836416, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 13211860992, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.12.attn_k.weight", "offset": 13211959296, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 13218643968, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 13244596224, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.12.attn_v.weight", "offset": 13260816384, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 13267501056, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 13613531136, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 13959561216, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 14305591296, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 14305615872, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 14305640448, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.13.attn_k.weight", "offset": 14305738752, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 14312423424, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 14338375680, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.13.attn_v.weight", "offset": 14354595840, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 14361280512, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 14707310592, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 15053340672, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 15399370752, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 15399395328, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 15399419904, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.14.attn_k.weight", "offset": 15399518208, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 15406202880, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 15432155136, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 15448375296, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 15455059968, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 15801090048, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 16147120128, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 16493150208, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 16493174784, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 16493199360, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.15.attn_k.weight", "offset": 16493297664, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 16499982336, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 16525934592, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 16542154752, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 16548839424, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 16894869504, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 17240899584, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 17586929664, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 17586954240, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 17586978816, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.16.attn_k.weight", "offset": 17587077120, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 17593761792, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 17619714048, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 17635934208, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 17642618880, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 17988648960, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 18334679040, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 18680709120, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 18680733696, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 18680758272, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.17.attn_k.weight", "offset": 18680856576, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 18687541248, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 18713493504, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.17.attn_v.weight", "offset": 18729713664, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 18736398336, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 19082428416, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 19428458496, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 19774488576, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 19774513152, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 19774537728, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.18.attn_k.weight", "offset": 19774636032, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 19781320704, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 19807272960, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 19823493120, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 19830177792, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 20176207872, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 20522237952, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 20868268032, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 20868292608, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 20868317184, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.19.attn_k.weight", "offset": 20868415488, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 20875100160, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 20901052416, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 20917272576, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 20923957248, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 21269987328, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 21616017408, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 21962047488, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 21962072064, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 21962096640, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.20.attn_k.weight", "offset": 21962194944, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 21968879616, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 21994831872, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 22011052032, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 22017736704, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 22363766784, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 22709796864, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 23055826944, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 23055851520, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 23055876096, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.21.attn_k.weight", "offset": 23055974400, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 23062659072, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 23088611328, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 23104831488, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 23111516160, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 23457546240, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 23803576320, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 24149606400, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 24149630976, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 24149655552, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.22.attn_q.weight", "offset": 24156340224, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 24172560384, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 24179245056, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 24525275136, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 24871305216, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 25217335296, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 25217433600, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 25217458176, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 25217482752, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.23.attn_k.weight", "offset": 25243435008, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.23.attn_q.weight", "offset": 25250119680, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 25266339840, "shape": [ 6144, 16384, 8 ], "size": 16220160, "type": 11 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 25612369920, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 25958400000, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 26304430080, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 26304528384, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 26304552960, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 26304577536, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.23.attn_v.weight", "offset": 26330529792, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.24.attn_q.weight", "offset": 26337214464, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 26353434624, "shape": [ 6144, 16384, 8 ], "size": 16220160, "type": 11 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 26699464704, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 27045494784, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 27391524864, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 27391623168, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 27391647744, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 27391672320, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 27398356992, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.24.attn_v.weight", "offset": 27424309248, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 27430993920, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 27777024000, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 28123054080, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.25.attn_k.weight", "offset": 28123152384, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 28129837056, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 28155789312, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 28172009472, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 28178694144, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 28524724224, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 28524748800, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 28524773376, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 11 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 28870803456, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.26.attn_k.weight", "offset": 28870901760, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 28877586432, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 28903538688, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 28919758848, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 28926443520, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 29272473600, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 29618503680, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 29618528256, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 29618552832, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.27.attn_k.weight", "offset": 29618651136, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 29625335808, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.27.attn_q.weight", "offset": 29651288064, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 29667508224, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 29674192896, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 30020222976, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 30366253056, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 30712283136, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 30712307712, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 30712332288, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.28.attn_k.weight", "offset": 30712430592, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 30719115264, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 30745067520, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 30761287680, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 30767972352, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 31114002432, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 31460032512, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 31806062592, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 31806087168, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 31806111744, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.29.attn_k.weight", "offset": 31806210048, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 31812894720, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 31838846976, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 31855067136, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 31861751808, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 32207781888, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 32553811968, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.29.attn_norm.weight", "offset": 32899842048, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 32899866624, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 32899891200, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.30.attn_k.weight", "offset": 32899989504, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 32906674176, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 32932626432, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 32948846592, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 32955531264, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 33301561344, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 33647591424, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.30.attn_norm.weight", "offset": 33993621504, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 33993646080, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 33993670656, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.31.attn_k.weight", "offset": 33993768960, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 34000453632, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.31.attn_q.weight", "offset": 34026405888, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 34042626048, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 34049310720, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 34395340800, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 34741370880, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.31.attn_norm.weight", "offset": 35087400960, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 35087425536, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 35087450112, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.32.attn_k.weight", "offset": 35087548416, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.32.attn_output.weight", "offset": 35094233088, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.32.attn_q.weight", "offset": 35120185344, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.32.attn_v.weight", "offset": 35136405504, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 35143090176, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 35489120256, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 35835150336, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.32.attn_norm.weight", "offset": 36181180416, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 36181204992, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 36181229568, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.33.attn_k.weight", "offset": 36181327872, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.33.attn_output.weight", "offset": 36188012544, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.33.attn_q.weight", "offset": 36213964800, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.33.attn_v.weight", "offset": 36230184960, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 36236869632, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 36582899712, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 36928929792, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.33.attn_norm.weight", "offset": 37274959872, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 37274984448, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 37275009024, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.34.attn_k.weight", "offset": 37275107328, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.34.attn_output.weight", "offset": 37281792000, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.34.attn_q.weight", "offset": 37307744256, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.34.attn_v.weight", "offset": 37323964416, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 37330649088, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 37676679168, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 38022709248, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.34.attn_norm.weight", "offset": 38368739328, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 38368763904, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 38368788480, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.35.attn_k.weight", "offset": 38368886784, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.35.attn_output.weight", "offset": 38375571456, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.35.attn_q.weight", "offset": 38401523712, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.35.attn_v.weight", "offset": 38417743872, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 38424428544, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 38770458624, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 39116488704, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.35.attn_norm.weight", "offset": 39462518784, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 39462543360, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 39462567936, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.36.attn_k.weight", "offset": 39462666240, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.36.attn_output.weight", "offset": 39469350912, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.36.attn_q.weight", "offset": 39495303168, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.36.attn_v.weight", "offset": 39511523328, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 39518208000, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 39864238080, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 40210268160, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.36.attn_norm.weight", "offset": 40556298240, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 40556322816, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 40556347392, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.37.attn_k.weight", "offset": 40556445696, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.37.attn_output.weight", "offset": 40563130368, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.37.attn_q.weight", "offset": 40589082624, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.37.attn_v.weight", "offset": 40605302784, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 40611987456, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 40958017536, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 41304047616, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.37.attn_norm.weight", "offset": 41650077696, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 41650102272, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 41650126848, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.38.attn_k.weight", "offset": 41650225152, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.38.attn_output.weight", "offset": 41656909824, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.38.attn_q.weight", "offset": 41682862080, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.38.attn_v.weight", "offset": 41699082240, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 41705766912, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 42051796992, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 42397827072, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.38.attn_norm.weight", "offset": 42743857152, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 42743881728, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 42743906304, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.39.attn_k.weight", "offset": 42744004608, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.39.attn_output.weight", "offset": 42750689280, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.39.attn_q.weight", "offset": 42776641536, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.39.attn_v.weight", "offset": 42792861696, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 42799546368, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 43145576448, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 43491606528, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.39.attn_norm.weight", "offset": 43837636608, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 43837661184, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 43837685760, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.40.attn_k.weight", "offset": 43837784064, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.40.attn_output.weight", "offset": 43844468736, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.40.attn_q.weight", "offset": 43870420992, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.40.attn_v.weight", "offset": 43886641152, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 43893325824, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 44239355904, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 44585385984, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.40.attn_norm.weight", "offset": 44931416064, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 44931440640, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 44931465216, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.41.attn_k.weight", "offset": 44931563520, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.41.attn_output.weight", "offset": 44938248192, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.41.attn_q.weight", "offset": 44964200448, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.41.attn_v.weight", "offset": 44980420608, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 44987105280, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 45333135360, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 45679165440, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.41.attn_norm.weight", "offset": 46025195520, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 46025220096, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 46025244672, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.42.attn_k.weight", "offset": 46025342976, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.42.attn_output.weight", "offset": 46032027648, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.42.attn_q.weight", "offset": 46057979904, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.42.attn_v.weight", "offset": 46074200064, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 46080884736, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 46426914816, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 46772944896, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.42.attn_norm.weight", "offset": 47118974976, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 47118999552, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 47119024128, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.43.attn_k.weight", "offset": 47119122432, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.43.attn_output.weight", "offset": 47125807104, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.43.attn_q.weight", "offset": 47151759360, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.43.attn_v.weight", "offset": 47167979520, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 47174664192, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 47520694272, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 47866724352, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.43.attn_norm.weight", "offset": 48212754432, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 48212779008, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 48212803584, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.44.attn_k.weight", "offset": 48212901888, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.44.attn_output.weight", "offset": 48219586560, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.44.attn_q.weight", "offset": 48245538816, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.44.attn_v.weight", "offset": 48261758976, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 48268443648, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 48614473728, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 48960503808, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.44.attn_norm.weight", "offset": 49306533888, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 49306558464, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 49306583040, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.45.attn_k.weight", "offset": 49306681344, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.45.attn_output.weight", "offset": 49313366016, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.45.attn_q.weight", "offset": 49339318272, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.45.attn_v.weight", "offset": 49355538432, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 49362223104, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 49708253184, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 50054283264, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.45.attn_norm.weight", "offset": 50400313344, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 50400337920, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 50400362496, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.46.attn_k.weight", "offset": 50400460800, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.46.attn_output.weight", "offset": 50407145472, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.46.attn_q.weight", "offset": 50433097728, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.46.attn_v.weight", "offset": 50449317888, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 50456002560, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 50802032640, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 51148062720, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.46.attn_norm.weight", "offset": 51494092800, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.46.ffn_norm.weight", "offset": 51494117376, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 51494141952, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.47.attn_k.weight", "offset": 51494240256, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.47.attn_output.weight", "offset": 51500924928, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.47.attn_q.weight", "offset": 51526877184, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.47.attn_v.weight", "offset": 51543097344, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 51549782016, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 51895812096, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 52241842176, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.47.attn_norm.weight", "offset": 52587872256, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.47.ffn_norm.weight", "offset": 52587896832, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 52587921408, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.48.attn_k.weight", "offset": 52588019712, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.48.attn_output.weight", "offset": 52594704384, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.48.attn_q.weight", "offset": 52620656640, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.48.attn_v.weight", "offset": 52636876800, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 52643561472, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 52989591552, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 53335621632, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.48.attn_norm.weight", "offset": 53681651712, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.48.ffn_norm.weight", "offset": 53681676288, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 53681700864, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.49.attn_q.weight", "offset": 53688385536, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.49.attn_v.weight", "offset": 53704605696, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 53711290368, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 54057320448, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 54403350528, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 54749380608, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.49.attn_norm.weight", "offset": 54749478912, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.49.ffn_norm.weight", "offset": 54749503488, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.49.attn_output.weight", "offset": 54749528064, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.50.attn_k.weight", "offset": 54775480320, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.50.attn_q.weight", "offset": 54782164992, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 54798385152, "shape": [ 6144, 16384, 8 ], "size": 16220160, "type": 11 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 55144415232, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 55490445312, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 55836475392, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.50.attn_norm.weight", "offset": 55836573696, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.50.ffn_norm.weight", "offset": 55836598272, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 55836622848, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.50.attn_v.weight", "offset": 55862575104, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.51.attn_q.weight", "offset": 55869259776, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 55885479936, "shape": [ 6144, 16384, 8 ], "size": 16220160, "type": 11 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 56231510016, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 56577540096, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 56923570176, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.51.attn_norm.weight", "offset": 56923668480, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.51.ffn_norm.weight", "offset": 56923693056, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 56923717632, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.51.attn_output.weight", "offset": 56930402304, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.51.attn_v.weight", "offset": 56956354560, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 56963039232, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 57309069312, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 57655099392, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.52.attn_k.weight", "offset": 57655197696, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.52.attn_output.weight", "offset": 57661882368, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.52.attn_q.weight", "offset": 57687834624, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.52.attn_v.weight", "offset": 57704054784, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 57710739456, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.52.attn_norm.weight", "offset": 58056769536, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.52.ffn_norm.weight", "offset": 58056794112, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 58056818688, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 11 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 58402848768, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.53.attn_k.weight", "offset": 58402947072, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.53.attn_output.weight", "offset": 58409631744, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.53.attn_q.weight", "offset": 58435584000, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.53.attn_v.weight", "offset": 58451804160, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 58458488832, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 58804518912, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.53.attn_norm.weight", "offset": 59150548992, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.53.ffn_norm.weight", "offset": 59150573568, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 59150598144, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.54.attn_k.weight", "offset": 59150696448, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.54.attn_output.weight", "offset": 59157381120, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.54.attn_q.weight", "offset": 59183333376, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.54.attn_v.weight", "offset": 59199553536, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 59206238208, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 59552268288, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 59898298368, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.54.attn_norm.weight", "offset": 60244328448, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.54.ffn_norm.weight", "offset": 60244353024, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 60244377600, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.55.attn_k.weight", "offset": 60244475904, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.55.attn_output.weight", "offset": 60251160576, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.55.attn_q.weight", "offset": 60277112832, "shape": [ 6144, 6144 ], "size": 25952256, "type": 11 }, { "name": "blk.55.attn_v.weight", "offset": 60293332992, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "output.weight", "offset": 60300017664, "shape": [ 6144, 32768 ], "size": 6684672, "type": 14 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 60465168384, "shape": [ 6144, 16384, 8 ], "size": 165150720, "type": 11 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 60811198464, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 61157228544, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 11 }, { "name": "blk.55.attn_norm.weight", "offset": 61503258624, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.55.ffn_norm.weight", "offset": 61503283200, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "output_norm.weight", "offset": 61503307776, "shape": [ 6144 ], "size": 24576, "type": 0 } ], "version": 3 }