mixtral

{ "metadata": { "general.architecture": "llama", "general.file_type": 10, "general.name": "Mixtral-8x22B-Instruct-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 65536, "llama.embedding_length": 6144, "llama.expert_count": 8, "llama.expert_used_count": 2, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.chat_template.tool_use": "{{bos_token}}{% set user_messages = messages | selectattr('role', 'equalto', 'user') | list %}{% for message in messages %}{% if message['role'] == 'user' %}{% if message == user_messages[-1] %}{% if tools %}{{'[AVAILABLE_TOOLS]'+ tools|string + '[/AVAILABLE_TOOLS]'}}{% endif %}{{ '[INST]' + message['content'] + '[/INST]' }}{% else %}{{ '[INST]' + message['content'] + '[/INST]' }}{% endif %}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + ' ' + eos_token}}{% elif message['role'] == 'tool_results' %}{{'[TOOL_RESULTS]' + message['content']|string + '[/TOOL_RESULTS]'}}{% elif message['role'] == 'tool_calls' %}{{'[TOOL_CALLS]' + message['content']|string + eos_token}}{% endif %}{% endfor %}", "tokenizer.chat_templates": "tool_use", "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 140630071296, "tensors": [ { "name": "blk.0.ffn_gate_inp.weight", "offset": 66060288, "shape": [ 6144, 8 ], "size": 66060288, "type": 1 }, { "name": "blk.0.attn_k.weight", "offset": 66158592, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 72843264, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 98795520, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.0.attn_v.weight", "offset": 111181824, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.0.ffn_gate_exps.weight", "offset": 117866496, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.0.ffn_down_exps.weight", "offset": 382107648, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.0.ffn_up_exps.weight", "offset": 728137728, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.0.attn_norm.weight", "offset": 992378880, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 992403456, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 992428032, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.1.attn_k.weight", "offset": 992526336, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 999211008, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 1025163264, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.1.attn_v.weight", "offset": 1037549568, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 1044234240, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 1308475392, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 1654505472, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.1.attn_norm.weight", "offset": 1918746624, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 1918771200, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 1918795776, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.2.attn_k.weight", "offset": 1918894080, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 1925578752, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 1951531008, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.2.attn_v.weight", "offset": 1963917312, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 1970601984, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 2234843136, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 2580873216, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.2.attn_norm.weight", "offset": 2845114368, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 2845138944, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 2845163520, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.3.attn_k.weight", "offset": 2845261824, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 2851946496, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 2877898752, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.3.attn_v.weight", "offset": 2890285056, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 2896969728, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 3161210880, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 3507240960, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.3.attn_norm.weight", "offset": 3771482112, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 3771506688, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 3771531264, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.4.attn_k.weight", "offset": 3771629568, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 3778314240, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 3804266496, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.4.attn_v.weight", "offset": 3816652800, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 3823337472, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 4087578624, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 4433608704, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.4.attn_norm.weight", "offset": 4697849856, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 4697874432, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 4697899008, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.5.attn_k.weight", "offset": 4697997312, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 4704681984, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 4730634240, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.5.attn_v.weight", "offset": 4743020544, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 4749705216, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 5013946368, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 5359976448, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.5.attn_norm.weight", "offset": 5624217600, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 5624242176, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 5624266752, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.6.attn_k.weight", "offset": 5624365056, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 5631049728, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 5657001984, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.6.attn_v.weight", "offset": 5669388288, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 5676072960, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 5940314112, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 6286344192, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.6.attn_norm.weight", "offset": 6550585344, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 6550609920, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 6550634496, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.7.attn_k.weight", "offset": 6550732800, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 6557417472, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 6583369728, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.7.attn_v.weight", "offset": 6595756032, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 6602440704, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 6866681856, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 7212711936, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.7.attn_norm.weight", "offset": 7476953088, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 7476977664, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 7477002240, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.8.attn_k.weight", "offset": 7477100544, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 7483785216, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 7509737472, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.8.attn_v.weight", "offset": 7522123776, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 7528808448, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 7793049600, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 8139079680, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.8.attn_norm.weight", "offset": 8403320832, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 8403345408, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 8403369984, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.9.attn_k.weight", "offset": 8403468288, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 8410152960, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 8436105216, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.9.attn_v.weight", "offset": 8448491520, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 8455176192, "shape": [ 6144, 8 ], "size": 6684672, "type": 1 }, { "name": "blk.10.attn_k.weight", "offset": 8455274496, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 8461959168, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 8487911424, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.10.attn_v.weight", "offset": 8500297728, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 8506982400, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 8771223552, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 9117253632, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.9.attn_norm.weight", "offset": 9381494784, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 9381519360, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 9381543936, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 10 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 9645785088, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 9991815168, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.10.attn_norm.weight", "offset": 10256056320, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 10256080896, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 10256105472, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.11.attn_k.weight", "offset": 10256203776, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 10262888448, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 10288840704, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.11.attn_v.weight", "offset": 10301227008, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 10307911680, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 10572152832, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 10918182912, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.11.attn_norm.weight", "offset": 11182424064, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 11182448640, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 11182473216, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.12.attn_k.weight", "offset": 11182571520, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 11189256192, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 11215208448, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.12.attn_v.weight", "offset": 11227594752, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 11234279424, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 11498520576, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 11844550656, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.12.attn_norm.weight", "offset": 12108791808, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 12108816384, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 12108840960, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.13.attn_k.weight", "offset": 12108939264, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 12115623936, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 12141576192, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.13.attn_v.weight", "offset": 12153962496, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 12160647168, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 12424888320, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 12770918400, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.13.attn_norm.weight", "offset": 13035159552, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 13035184128, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 13035208704, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.14.attn_k.weight", "offset": 13035307008, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 13041991680, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 13067943936, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.14.attn_v.weight", "offset": 13080330240, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 13087014912, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 13351256064, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 13697286144, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.14.attn_norm.weight", "offset": 13961527296, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 13961551872, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 13961576448, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.15.attn_k.weight", "offset": 13961674752, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 13968359424, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 13994311680, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.15.attn_v.weight", "offset": 14006697984, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 14013382656, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 14277623808, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 14623653888, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.15.attn_norm.weight", "offset": 14887895040, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 14887919616, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 14887944192, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.16.attn_k.weight", "offset": 14888042496, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 14894727168, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 14920679424, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.16.attn_v.weight", "offset": 14933065728, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 14939750400, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 15203991552, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 15550021632, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.16.attn_norm.weight", "offset": 15814262784, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 15814287360, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 15814311936, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.17.attn_k.weight", "offset": 15814410240, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 15821094912, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 15847047168, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.17.attn_v.weight", "offset": 15859433472, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 15866118144, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 16130359296, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 16476389376, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.17.attn_norm.weight", "offset": 16740630528, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 16740655104, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 16740679680, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.18.attn_k.weight", "offset": 16740777984, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 16747462656, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 16773414912, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.18.attn_v.weight", "offset": 16785801216, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 16792485888, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 17056727040, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 17402757120, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.18.attn_norm.weight", "offset": 17666998272, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 17667022848, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 17667047424, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.19.attn_k.weight", "offset": 17667145728, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 17673830400, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 17699782656, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.19.attn_v.weight", "offset": 17712168960, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 17718853632, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 17983094784, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 18329124864, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.19.attn_norm.weight", "offset": 18593366016, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 18593390592, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 18593415168, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.20.attn_k.weight", "offset": 18593513472, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 18600198144, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 18626150400, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.20.attn_v.weight", "offset": 18638536704, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 18645221376, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 18909462528, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 19255492608, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.20.attn_norm.weight", "offset": 19519733760, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 19519758336, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 19519782912, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.21.attn_k.weight", "offset": 19519881216, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 19526565888, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 19552518144, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.21.attn_v.weight", "offset": 19564904448, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 19571589120, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 19835830272, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 20181860352, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.21.attn_norm.weight", "offset": 20446101504, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 20446126080, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 20446150656, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.22.attn_q.weight", "offset": 20452835328, "shape": [ 6144, 6144 ], "size": 6684672, "type": 10 }, { "name": "blk.22.attn_v.weight", "offset": 20465221632, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 20471906304, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 20736147456, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 21082177536, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 21346418688, "shape": [ 6144, 8 ], "size": 264241152, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 21346516992, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 21346541568, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 21346566144, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.23.attn_k.weight", "offset": 21372518400, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.23.attn_q.weight", "offset": 21379203072, "shape": [ 6144, 6144 ], "size": 6684672, "type": 10 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 21391589376, "shape": [ 6144, 16384, 8 ], "size": 12386304, "type": 10 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 21655830528, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 22001860608, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 22266101760, "shape": [ 6144, 8 ], "size": 264241152, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 22266200064, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 22266224640, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 22266249216, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.23.attn_v.weight", "offset": 22292201472, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.24.attn_q.weight", "offset": 22298886144, "shape": [ 6144, 6144 ], "size": 6684672, "type": 10 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 22311272448, "shape": [ 6144, 16384, 8 ], "size": 12386304, "type": 10 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 22575513600, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 22921543680, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 23185784832, "shape": [ 6144, 8 ], "size": 264241152, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 23185883136, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 23185907712, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 23185932288, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 23192616960, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.24.attn_v.weight", "offset": 23218569216, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 23225253888, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 23489495040, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 23835525120, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.25.attn_k.weight", "offset": 23835623424, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 23842308096, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 23868260352, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.25.attn_v.weight", "offset": 23880646656, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 23887331328, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.25.attn_norm.weight", "offset": 24151572480, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 24151597056, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 24151621632, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 10 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 24415862784, "shape": [ 6144, 8 ], "size": 264241152, "type": 1 }, { "name": "blk.26.attn_k.weight", "offset": 24415961088, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 24422645760, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 24448598016, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.26.attn_v.weight", "offset": 24460984320, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 24467668992, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 24813699072, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.26.attn_norm.weight", "offset": 25077940224, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 25077964800, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 25077989376, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.27.attn_k.weight", "offset": 25078087680, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 25084772352, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.27.attn_q.weight", "offset": 25110724608, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.27.attn_v.weight", "offset": 25123110912, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 25129795584, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 25394036736, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 25740066816, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.27.attn_norm.weight", "offset": 26004307968, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 26004332544, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 26004357120, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.28.attn_k.weight", "offset": 26004455424, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 26011140096, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 26037092352, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.28.attn_v.weight", "offset": 26049478656, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 26056163328, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 26320404480, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 26666434560, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.28.attn_norm.weight", "offset": 26930675712, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 26930700288, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 26930724864, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.29.attn_k.weight", "offset": 26930823168, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 26937507840, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 26963460096, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.29.attn_v.weight", "offset": 26975846400, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 26982531072, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 27246772224, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 27592802304, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.29.attn_norm.weight", "offset": 27857043456, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 27857068032, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 27857092608, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.30.attn_k.weight", "offset": 27857190912, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 27863875584, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 27889827840, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.30.attn_v.weight", "offset": 27902214144, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 27908898816, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 28173139968, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 28519170048, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.30.attn_norm.weight", "offset": 28783411200, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 28783435776, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 28783460352, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.31.attn_k.weight", "offset": 28783558656, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 28790243328, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.31.attn_q.weight", "offset": 28816195584, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.31.attn_v.weight", "offset": 28828581888, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 28835266560, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 29099507712, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 29445537792, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.31.attn_norm.weight", "offset": 29709778944, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 29709803520, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 29709828096, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.32.attn_k.weight", "offset": 29709926400, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.32.attn_output.weight", "offset": 29716611072, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.32.attn_q.weight", "offset": 29742563328, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.32.attn_v.weight", "offset": 29754949632, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 29761634304, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 30025875456, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 30371905536, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.32.attn_norm.weight", "offset": 30636146688, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 30636171264, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 30636195840, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.33.attn_k.weight", "offset": 30636294144, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.33.attn_output.weight", "offset": 30642978816, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.33.attn_q.weight", "offset": 30668931072, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.33.attn_v.weight", "offset": 30681317376, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 30688002048, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 30952243200, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 31298273280, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.33.attn_norm.weight", "offset": 31562514432, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 31562539008, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 31562563584, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.34.attn_k.weight", "offset": 31562661888, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.34.attn_output.weight", "offset": 31569346560, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.34.attn_q.weight", "offset": 31595298816, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.34.attn_v.weight", "offset": 31607685120, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 31614369792, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 31878610944, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 32224641024, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.34.attn_norm.weight", "offset": 32488882176, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 32488906752, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 32488931328, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.35.attn_k.weight", "offset": 32489029632, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.35.attn_output.weight", "offset": 32495714304, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.35.attn_q.weight", "offset": 32521666560, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.35.attn_v.weight", "offset": 32534052864, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 32540737536, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 32804978688, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 33151008768, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.35.attn_norm.weight", "offset": 33415249920, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 33415274496, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 33415299072, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.36.attn_k.weight", "offset": 33415397376, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.36.attn_output.weight", "offset": 33422082048, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.36.attn_q.weight", "offset": 33448034304, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.36.attn_v.weight", "offset": 33460420608, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 33467105280, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 33731346432, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 34077376512, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.36.attn_norm.weight", "offset": 34341617664, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 34341642240, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 34341666816, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.37.attn_k.weight", "offset": 34341765120, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.37.attn_output.weight", "offset": 34348449792, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.37.attn_q.weight", "offset": 34374402048, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.37.attn_v.weight", "offset": 34386788352, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 34393473024, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 34657714176, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 35003744256, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.37.attn_norm.weight", "offset": 35267985408, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 35268009984, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 35268034560, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.38.attn_k.weight", "offset": 35268132864, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.38.attn_output.weight", "offset": 35274817536, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.38.attn_q.weight", "offset": 35300769792, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.38.attn_v.weight", "offset": 35313156096, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 35319840768, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 35584081920, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 35930112000, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.38.attn_norm.weight", "offset": 36194353152, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 36194377728, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 36194402304, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.39.attn_k.weight", "offset": 36194500608, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.39.attn_output.weight", "offset": 36201185280, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.39.attn_q.weight", "offset": 36227137536, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.39.attn_v.weight", "offset": 36239523840, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 36246208512, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 36510449664, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 36856479744, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.39.attn_norm.weight", "offset": 37120720896, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 37120745472, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 37120770048, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.40.attn_k.weight", "offset": 37120868352, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.40.attn_output.weight", "offset": 37127553024, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.40.attn_q.weight", "offset": 37153505280, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.40.attn_v.weight", "offset": 37165891584, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 37172576256, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 37436817408, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 37782847488, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.40.attn_norm.weight", "offset": 38047088640, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 38047113216, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 38047137792, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.41.attn_k.weight", "offset": 38047236096, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.41.attn_output.weight", "offset": 38053920768, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.41.attn_q.weight", "offset": 38079873024, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.41.attn_v.weight", "offset": 38092259328, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 38098944000, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 38363185152, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 38709215232, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.41.attn_norm.weight", "offset": 38973456384, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 38973480960, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 38973505536, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.42.attn_k.weight", "offset": 38973603840, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.42.attn_output.weight", "offset": 38980288512, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.42.attn_q.weight", "offset": 39006240768, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.42.attn_v.weight", "offset": 39018627072, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 39025311744, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 39289552896, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 39635582976, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.42.attn_norm.weight", "offset": 39899824128, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 39899848704, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 39899873280, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.43.attn_k.weight", "offset": 39899971584, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.43.attn_output.weight", "offset": 39906656256, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.43.attn_q.weight", "offset": 39932608512, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.43.attn_v.weight", "offset": 39944994816, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 39951679488, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 40215920640, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 40561950720, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.43.attn_norm.weight", "offset": 40826191872, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 40826216448, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 40826241024, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.44.attn_k.weight", "offset": 40826339328, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.44.attn_output.weight", "offset": 40833024000, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.44.attn_q.weight", "offset": 40858976256, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.44.attn_v.weight", "offset": 40871362560, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 40878047232, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 41142288384, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 41488318464, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.44.attn_norm.weight", "offset": 41752559616, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 41752584192, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 41752608768, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.45.attn_k.weight", "offset": 41752707072, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.45.attn_output.weight", "offset": 41759391744, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.45.attn_q.weight", "offset": 41785344000, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.45.attn_v.weight", "offset": 41797730304, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 41804414976, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 42068656128, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 42414686208, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.45.attn_norm.weight", "offset": 42678927360, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 42678951936, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 42678976512, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.46.attn_k.weight", "offset": 42679074816, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.46.attn_output.weight", "offset": 42685759488, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.46.attn_q.weight", "offset": 42711711744, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.46.attn_v.weight", "offset": 42724098048, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 42730782720, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 42995023872, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 43341053952, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.46.attn_norm.weight", "offset": 43605295104, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.46.ffn_norm.weight", "offset": 43605319680, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 43605344256, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.47.attn_k.weight", "offset": 43605442560, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.47.attn_output.weight", "offset": 43612127232, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.47.attn_q.weight", "offset": 43638079488, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.47.attn_v.weight", "offset": 43650465792, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 43657150464, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 43921391616, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 44267421696, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.47.attn_norm.weight", "offset": 44531662848, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.47.ffn_norm.weight", "offset": 44531687424, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 44531712000, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.48.attn_k.weight", "offset": 44531810304, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.48.attn_output.weight", "offset": 44538494976, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.48.attn_q.weight", "offset": 44564447232, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.48.attn_v.weight", "offset": 44576833536, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 44583518208, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 44847759360, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 45193789440, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.48.attn_norm.weight", "offset": 45458030592, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.48.ffn_norm.weight", "offset": 45458055168, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 45458079744, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.49.attn_q.weight", "offset": 45464764416, "shape": [ 6144, 6144 ], "size": 6684672, "type": 10 }, { "name": "blk.49.attn_v.weight", "offset": 45477150720, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 45483835392, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 45748076544, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 46094106624, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 46358347776, "shape": [ 6144, 8 ], "size": 264241152, "type": 1 }, { "name": "blk.49.attn_norm.weight", "offset": 46358446080, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.49.ffn_norm.weight", "offset": 46358470656, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.49.attn_output.weight", "offset": 46358495232, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.50.attn_k.weight", "offset": 46384447488, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.50.attn_q.weight", "offset": 46391132160, "shape": [ 6144, 6144 ], "size": 6684672, "type": 10 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 46403518464, "shape": [ 6144, 16384, 8 ], "size": 12386304, "type": 10 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 46667759616, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 47013789696, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 47278030848, "shape": [ 6144, 8 ], "size": 264241152, "type": 1 }, { "name": "blk.50.attn_norm.weight", "offset": 47278129152, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.50.ffn_norm.weight", "offset": 47278153728, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 47278178304, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.50.attn_v.weight", "offset": 47304130560, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.51.attn_q.weight", "offset": 47310815232, "shape": [ 6144, 6144 ], "size": 6684672, "type": 10 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 47323201536, "shape": [ 6144, 16384, 8 ], "size": 12386304, "type": 10 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 47587442688, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 47933472768, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 48197713920, "shape": [ 6144, 8 ], "size": 264241152, "type": 1 }, { "name": "blk.51.attn_norm.weight", "offset": 48197812224, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.51.ffn_norm.weight", "offset": 48197836800, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 48197861376, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.51.attn_output.weight", "offset": 48204546048, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.51.attn_v.weight", "offset": 48230498304, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 48237182976, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 48501424128, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 48847454208, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.52.attn_k.weight", "offset": 48847552512, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.52.attn_output.weight", "offset": 48854237184, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.52.attn_q.weight", "offset": 48880189440, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.52.attn_v.weight", "offset": 48892575744, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 48899260416, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.52.attn_norm.weight", "offset": 49163501568, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.52.ffn_norm.weight", "offset": 49163526144, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 49163550720, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 10 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 49427791872, "shape": [ 6144, 8 ], "size": 264241152, "type": 1 }, { "name": "blk.53.attn_k.weight", "offset": 49427890176, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.53.attn_output.weight", "offset": 49434574848, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.53.attn_q.weight", "offset": 49460527104, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.53.attn_v.weight", "offset": 49472913408, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 49479598080, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 49825628160, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.53.attn_norm.weight", "offset": 50089869312, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.53.ffn_norm.weight", "offset": 50089893888, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 50089918464, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.54.attn_k.weight", "offset": 50090016768, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.54.attn_output.weight", "offset": 50096701440, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.54.attn_q.weight", "offset": 50122653696, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.54.attn_v.weight", "offset": 50135040000, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 50141724672, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 10 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 50405965824, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 50751995904, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.54.attn_norm.weight", "offset": 51016237056, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.54.ffn_norm.weight", "offset": 51016261632, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 51016286208, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.55.attn_k.weight", "offset": 51016384512, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.55.attn_output.weight", "offset": 51023069184, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.55.attn_q.weight", "offset": 51049021440, "shape": [ 6144, 6144 ], "size": 25952256, "type": 10 }, { "name": "blk.55.attn_v.weight", "offset": 51061407744, "shape": [ 6144, 1024 ], "size": 12386304, "type": 8 }, { "name": "output.weight", "offset": 51068092416, "shape": [ 6144, 32768 ], "size": 6684672, "type": 14 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 51233243136, "shape": [ 6144, 16384, 8 ], "size": 165150720, "type": 10 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 51497484288, "shape": [ 16384, 6144, 8 ], "size": 264241152, "type": 11 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 51843514368, "shape": [ 6144, 16384, 8 ], "size": 346030080, "type": 10 }, { "name": "blk.55.attn_norm.weight", "offset": 52107755520, "shape": [ 6144 ], "size": 264241152, "type": 0 }, { "name": "blk.55.ffn_norm.weight", "offset": 52107780096, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "output_norm.weight", "offset": 52107804672, "shape": [ 6144 ], "size": 24576, "type": 0 } ], "version": 3 }