mixtral

{ "metadata": { "general.architecture": "llama", "general.file_type": 13, "general.name": "Mixtral-8x22B-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 65536, "llama.embedding_length": 6144, "llama.expert_count": 8, "llama.expert_used_count": 2, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32000, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32000 values)", "tokenizer.ggml.token_type": "... (32000 values)", "tokenizer.ggml.tokens": "... (32000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 140620634112, "tensors": [ { "name": "blk.0.ffn_gate_exps.weight", "offset": 84480000, "shape": [ 6144, 16384, 8 ], "size": 84480000, "type": 11 }, { "name": "blk.0.ffn_gate_inp.weight", "offset": 430510080, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.0.attn_k.weight", "offset": 430608384, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 437293056, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.0.attn_q.weight", "offset": 453513216, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 469733376, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.0.ffn_down_exps.weight", "offset": 476418048, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.0.ffn_up_exps.weight", "offset": 1030066176, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.0.attn_norm.weight", "offset": 1376096256, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 1376120832, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1376145408, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.1.attn_k.weight", "offset": 1376243712, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 1382928384, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.1.attn_q.weight", "offset": 1399148544, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 1415368704, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 1422053376, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 1768083456, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 2321731584, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 2667761664, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 2667786240, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 2667810816, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.2.attn_k.weight", "offset": 2667909120, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 2674593792, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.2.attn_q.weight", "offset": 2690813952, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.2.attn_v.weight", "offset": 2707034112, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 2713718784, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 3059748864, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 3613396992, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 3959427072, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 3959451648, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 3959476224, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.3.attn_k.weight", "offset": 3959574528, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 3966259200, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.3.attn_q.weight", "offset": 3982479360, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.3.attn_v.weight", "offset": 3998699520, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 4005384192, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 4351414272, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 4905062400, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 5251092480, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 5251117056, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 5251141632, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.4.attn_k.weight", "offset": 5251239936, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 5257924608, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.4.attn_q.weight", "offset": 5274144768, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 5290364928, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 5297049600, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 5643079680, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 6196727808, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 6542757888, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 6542782464, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 6542807040, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.5.attn_k.weight", "offset": 6542905344, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 6549590016, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.5.attn_q.weight", "offset": 6565810176, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 6582030336, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 6588715008, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 6934745088, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 7488393216, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 7834423296, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 7834447872, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 7834472448, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.6.attn_k.weight", "offset": 7834570752, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 7841255424, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.6.attn_q.weight", "offset": 7857475584, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 7873695744, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 7880380416, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 8226410496, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 8780058624, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 9126088704, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 9126113280, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 9126137856, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.7.attn_k.weight", "offset": 9126236160, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 9132920832, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.7.attn_q.weight", "offset": 9149140992, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 9165361152, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 9172045824, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 9518075904, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 10071724032, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.7.attn_norm.weight", "offset": 10417754112, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 10417778688, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 10417803264, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.8.attn_k.weight", "offset": 10417901568, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 10424586240, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.8.attn_q.weight", "offset": 10440806400, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 10457026560, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 10463711232, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 10809741312, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 11363389440, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 11709419520, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 11709444096, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 11709468672, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.9.attn_k.weight", "offset": 11709566976, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 11716251648, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.9.attn_q.weight", "offset": 11732471808, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.9.attn_v.weight", "offset": 11748691968, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 11755376640, "shape": [ 6144, 8 ], "size": 6684672, "type": 1 }, { "name": "blk.10.attn_k.weight", "offset": 11755474944, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 11762159616, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.10.attn_q.weight", "offset": 11778379776, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 11794599936, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 11801284608, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 12147314688, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 12700962816, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 13046992896, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 13047017472, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 13047042048, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 11 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 13393072128, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 13946720256, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.10.attn_norm.weight", "offset": 14292750336, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 14292774912, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 14292799488, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.11.attn_k.weight", "offset": 14292897792, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 14299582464, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.11.attn_q.weight", "offset": 14315802624, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.11.attn_v.weight", "offset": 14332022784, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 14338707456, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 14684737536, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 15238385664, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 15584415744, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 15584440320, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 15584464896, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.12.attn_k.weight", "offset": 15584563200, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 15591247872, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.12.attn_q.weight", "offset": 15607468032, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.12.attn_v.weight", "offset": 15623688192, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 15630372864, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 15976402944, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 16530051072, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 16876081152, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 16876105728, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 16876130304, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.13.attn_k.weight", "offset": 16876228608, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 16882913280, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.13.attn_q.weight", "offset": 16899133440, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.13.attn_v.weight", "offset": 16915353600, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 16922038272, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 17268068352, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 17821716480, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 18167746560, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 18167771136, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 18167795712, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.14.attn_k.weight", "offset": 18167894016, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 18174578688, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.14.attn_q.weight", "offset": 18190798848, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 18207019008, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 18213703680, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 18559733760, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 19113381888, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 19459411968, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 19459436544, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 19459461120, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.15.attn_k.weight", "offset": 19459559424, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 19466244096, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.15.attn_q.weight", "offset": 19482464256, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 19498684416, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 19505369088, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 19851399168, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 20405047296, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 20751077376, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 20751101952, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 20751126528, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.16.attn_k.weight", "offset": 20751224832, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 20757909504, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.16.attn_q.weight", "offset": 20774129664, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 20790349824, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 20797034496, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 21143064576, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 21696712704, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 22042742784, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 22042767360, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 22042791936, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.17.attn_k.weight", "offset": 22042890240, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 22049574912, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.17.attn_q.weight", "offset": 22065795072, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.17.attn_v.weight", "offset": 22082015232, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 22088699904, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 22434729984, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 22988378112, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 23334408192, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 23334432768, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 23334457344, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.18.attn_k.weight", "offset": 23334555648, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 23341240320, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.18.attn_q.weight", "offset": 23357460480, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 23373680640, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 23380365312, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 23726395392, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 24280043520, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 24626073600, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 24626098176, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 24626122752, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.19.attn_k.weight", "offset": 24626221056, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 24632905728, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.19.attn_q.weight", "offset": 24649125888, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 24665346048, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 24672030720, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 25018060800, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 25571708928, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 25917739008, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 25917763584, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 25917788160, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.20.attn_k.weight", "offset": 25917886464, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 25924571136, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.20.attn_q.weight", "offset": 25940791296, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 25957011456, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 25963696128, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 26309726208, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 26863374336, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 27209404416, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 27209428992, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 27209453568, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.21.attn_k.weight", "offset": 27209551872, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 27216236544, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.21.attn_q.weight", "offset": 27232456704, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 27248676864, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 27255361536, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 27601391616, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 28155039744, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 28501069824, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 28501094400, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 28501118976, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.22.attn_k.weight", "offset": 28501217280, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 28507901952, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.22.attn_q.weight", "offset": 28524122112, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 28540342272, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 28547026944, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 28893057024, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 29446705152, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 29792735232, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 29792759808, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 29792784384, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.23.attn_q.weight", "offset": 29799469056, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 29815689216, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 29822373888, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 30168403968, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 30722052096, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 31068082176, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 31068180480, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 31068205056, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 31068229632, "shape": [ 6144, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.24.attn_k.weight", "offset": 31084449792, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.24.attn_q.weight", "offset": 31091134464, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 31107354624, "shape": [ 6144, 16384, 8 ], "size": 16220160, "type": 11 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 31453384704, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 32007032832, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 32353062912, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 32353161216, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 32353185792, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 32353210368, "shape": [ 6144, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 32369430528, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.25.attn_q.weight", "offset": 32376115200, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 32392335360, "shape": [ 6144, 16384, 8 ], "size": 16220160, "type": 11 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 32738365440, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 33292013568, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 33638043648, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.25.attn_norm.weight", "offset": 33638141952, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 33638166528, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 33638191104, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 33644875776, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 33661095936, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 33667780608, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 34013810688, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 34567458816, "shape": [ 6144, 8 ], "size": 553648128, "type": 1 }, { "name": "blk.26.attn_k.weight", "offset": 34567557120, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 34574241792, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.26.attn_q.weight", "offset": 34590461952, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 34606682112, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 34613366784, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 34959396864, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 34959421440, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 34959446016, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 11 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 35305476096, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.27.attn_k.weight", "offset": 35305574400, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 35312259072, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.27.attn_q.weight", "offset": 35328479232, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 35344699392, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 35351384064, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 35905032192, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 36251062272, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 36251086848, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 36251111424, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.28.attn_k.weight", "offset": 36251209728, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 36257894400, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.28.attn_q.weight", "offset": 36274114560, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 36290334720, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 36297019392, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 36643049472, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 37196697600, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 37542727680, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 37542752256, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 37542776832, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.29.attn_k.weight", "offset": 37542875136, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 37549559808, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.29.attn_q.weight", "offset": 37565779968, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 37582000128, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 37588684800, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 37934714880, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 38488363008, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.29.attn_norm.weight", "offset": 38834393088, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 38834417664, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 38834442240, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.30.attn_k.weight", "offset": 38834540544, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 38841225216, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.30.attn_q.weight", "offset": 38857445376, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 38873665536, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 38880350208, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 39226380288, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 39780028416, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.30.attn_norm.weight", "offset": 40126058496, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 40126083072, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 40126107648, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.31.attn_k.weight", "offset": 40126205952, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 40132890624, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.31.attn_q.weight", "offset": 40149110784, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 40165330944, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 40172015616, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 40518045696, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 41071693824, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.31.attn_norm.weight", "offset": 41417723904, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 41417748480, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 41417773056, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.32.attn_k.weight", "offset": 41417871360, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.32.attn_output.weight", "offset": 41424556032, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.32.attn_q.weight", "offset": 41440776192, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.32.attn_v.weight", "offset": 41456996352, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 41463681024, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 41809711104, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 42363359232, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.32.attn_norm.weight", "offset": 42709389312, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 42709413888, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 42709438464, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.33.attn_k.weight", "offset": 42709536768, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.33.attn_output.weight", "offset": 42716221440, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.33.attn_q.weight", "offset": 42732441600, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.33.attn_v.weight", "offset": 42748661760, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 42755346432, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 43101376512, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 43655024640, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.33.attn_norm.weight", "offset": 44001054720, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 44001079296, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 44001103872, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.34.attn_k.weight", "offset": 44001202176, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.34.attn_output.weight", "offset": 44007886848, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.34.attn_q.weight", "offset": 44024107008, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.34.attn_v.weight", "offset": 44040327168, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 44047011840, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 44393041920, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 44946690048, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.34.attn_norm.weight", "offset": 45292720128, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 45292744704, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 45292769280, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.35.attn_k.weight", "offset": 45292867584, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.35.attn_output.weight", "offset": 45299552256, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.35.attn_q.weight", "offset": 45315772416, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.35.attn_v.weight", "offset": 45331992576, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 45338677248, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 45684707328, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 46238355456, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.35.attn_norm.weight", "offset": 46584385536, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 46584410112, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 46584434688, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.36.attn_k.weight", "offset": 46584532992, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.36.attn_output.weight", "offset": 46591217664, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.36.attn_q.weight", "offset": 46607437824, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.36.attn_v.weight", "offset": 46623657984, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 46630342656, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 46976372736, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 47530020864, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.36.attn_norm.weight", "offset": 47876050944, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 47876075520, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 47876100096, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.37.attn_k.weight", "offset": 47876198400, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.37.attn_output.weight", "offset": 47882883072, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.37.attn_q.weight", "offset": 47899103232, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.37.attn_v.weight", "offset": 47915323392, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 47922008064, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 48268038144, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 48821686272, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.37.attn_norm.weight", "offset": 49167716352, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 49167740928, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 49167765504, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.38.attn_k.weight", "offset": 49167863808, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.38.attn_output.weight", "offset": 49174548480, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.38.attn_q.weight", "offset": 49190768640, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.38.attn_v.weight", "offset": 49206988800, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 49213673472, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 49559703552, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 50113351680, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.38.attn_norm.weight", "offset": 50459381760, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 50459406336, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 50459430912, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.39.attn_k.weight", "offset": 50459529216, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.39.attn_output.weight", "offset": 50466213888, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.39.attn_q.weight", "offset": 50482434048, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.39.attn_v.weight", "offset": 50498654208, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 50505338880, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 50851368960, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 51405017088, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.39.attn_norm.weight", "offset": 51751047168, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 51751071744, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 51751096320, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.40.attn_k.weight", "offset": 51751194624, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.40.attn_output.weight", "offset": 51757879296, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.40.attn_q.weight", "offset": 51774099456, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.40.attn_v.weight", "offset": 51790319616, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 51797004288, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 52143034368, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 52696682496, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.40.attn_norm.weight", "offset": 53042712576, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 53042737152, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 53042761728, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.41.attn_k.weight", "offset": 53042860032, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.41.attn_output.weight", "offset": 53049544704, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.41.attn_q.weight", "offset": 53065764864, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.41.attn_v.weight", "offset": 53081985024, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 53088669696, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 53434699776, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 53988347904, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.41.attn_norm.weight", "offset": 54334377984, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 54334402560, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 54334427136, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.42.attn_k.weight", "offset": 54334525440, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.42.attn_output.weight", "offset": 54341210112, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.42.attn_q.weight", "offset": 54357430272, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.42.attn_v.weight", "offset": 54373650432, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 54380335104, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 54726365184, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 55280013312, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.42.attn_norm.weight", "offset": 55626043392, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 55626067968, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 55626092544, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.43.attn_k.weight", "offset": 55626190848, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.43.attn_output.weight", "offset": 55632875520, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.43.attn_q.weight", "offset": 55649095680, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.43.attn_v.weight", "offset": 55665315840, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 55672000512, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 56018030592, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 56571678720, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.43.attn_norm.weight", "offset": 56917708800, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 56917733376, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 56917757952, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.44.attn_k.weight", "offset": 56917856256, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.44.attn_output.weight", "offset": 56924540928, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.44.attn_q.weight", "offset": 56940761088, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.44.attn_v.weight", "offset": 56956981248, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 56963665920, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 57309696000, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 57863344128, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.44.attn_norm.weight", "offset": 58209374208, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 58209398784, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 58209423360, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.45.attn_k.weight", "offset": 58209521664, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.45.attn_output.weight", "offset": 58216206336, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.45.attn_q.weight", "offset": 58232426496, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.45.attn_v.weight", "offset": 58248646656, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 58255331328, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 58601361408, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 59155009536, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.45.attn_norm.weight", "offset": 59501039616, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 59501064192, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 59501088768, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.46.attn_k.weight", "offset": 59501187072, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.46.attn_output.weight", "offset": 59507871744, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.46.attn_q.weight", "offset": 59524091904, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.46.attn_v.weight", "offset": 59540312064, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 59546996736, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 59893026816, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 60446674944, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.46.attn_norm.weight", "offset": 60792705024, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.46.ffn_norm.weight", "offset": 60792729600, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 60792754176, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.47.attn_k.weight", "offset": 60792852480, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.47.attn_output.weight", "offset": 60799537152, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.47.attn_q.weight", "offset": 60815757312, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.47.attn_v.weight", "offset": 60831977472, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 60838662144, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 61184692224, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 61738340352, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.47.attn_norm.weight", "offset": 62084370432, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.47.ffn_norm.weight", "offset": 62084395008, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 62084419584, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.48.attn_k.weight", "offset": 62084517888, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.48.attn_output.weight", "offset": 62091202560, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.48.attn_q.weight", "offset": 62107422720, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.48.attn_v.weight", "offset": 62123642880, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 62130327552, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 62476357632, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 63030005760, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.48.attn_norm.weight", "offset": 63376035840, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.48.ffn_norm.weight", "offset": 63376060416, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 63376084992, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.49.attn_k.weight", "offset": 63376183296, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.49.attn_output.weight", "offset": 63382867968, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.49.attn_q.weight", "offset": 63399088128, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.49.attn_v.weight", "offset": 63415308288, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 63421992960, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 63768023040, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 64321671168, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.49.attn_norm.weight", "offset": 64667701248, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.49.ffn_norm.weight", "offset": 64667725824, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 64667750400, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.50.attn_q.weight", "offset": 64674435072, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.50.attn_v.weight", "offset": 64690655232, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 64697339904, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 65043369984, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 65597018112, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 65943048192, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.50.attn_norm.weight", "offset": 65943146496, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.50.ffn_norm.weight", "offset": 65943171072, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 65943195648, "shape": [ 6144, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.51.attn_k.weight", "offset": 65959415808, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.51.attn_q.weight", "offset": 65966100480, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 65982320640, "shape": [ 6144, 16384, 8 ], "size": 16220160, "type": 11 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 66328350720, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 66881998848, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 67228028928, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.51.attn_norm.weight", "offset": 67228127232, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.51.ffn_norm.weight", "offset": 67228151808, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.51.attn_output.weight", "offset": 67228176384, "shape": [ 6144, 6144 ], "size": 24576, "type": 11 }, { "name": "blk.51.attn_v.weight", "offset": 67244396544, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.52.attn_q.weight", "offset": 67251081216, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 67267301376, "shape": [ 6144, 16384, 8 ], "size": 16220160, "type": 11 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 67613331456, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 68166979584, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 68513009664, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.52.attn_norm.weight", "offset": 68513107968, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.52.ffn_norm.weight", "offset": 68513132544, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 68513157120, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.52.attn_output.weight", "offset": 68519841792, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.52.attn_v.weight", "offset": 68536061952, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 68542746624, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 68888776704, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 69442424832, "shape": [ 6144, 8 ], "size": 553648128, "type": 1 }, { "name": "blk.53.attn_k.weight", "offset": 69442523136, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.53.attn_output.weight", "offset": 69449207808, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.53.attn_q.weight", "offset": 69465427968, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.53.attn_v.weight", "offset": 69481648128, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 69488332800, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 11 }, { "name": "blk.53.attn_norm.weight", "offset": 69834362880, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.53.ffn_norm.weight", "offset": 69834387456, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 69834412032, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 11 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 70180442112, "shape": [ 6144, 8 ], "size": 346030080, "type": 1 }, { "name": "blk.54.attn_k.weight", "offset": 70180540416, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.54.attn_output.weight", "offset": 70187225088, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.54.attn_q.weight", "offset": 70203445248, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.54.attn_v.weight", "offset": 70219665408, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 70226350080, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 70779998208, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.54.attn_norm.weight", "offset": 71126028288, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.54.ffn_norm.weight", "offset": 71126052864, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 71126077440, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.55.attn_k.weight", "offset": 71126175744, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.55.attn_output.weight", "offset": 71132860416, "shape": [ 6144, 6144 ], "size": 6684672, "type": 11 }, { "name": "blk.55.attn_q.weight", "offset": 71149080576, "shape": [ 6144, 6144 ], "size": 16220160, "type": 11 }, { "name": "blk.55.attn_v.weight", "offset": 71165300736, "shape": [ 6144, 1024 ], "size": 16220160, "type": 8 }, { "name": "output.weight", "offset": 71171985408, "shape": [ 6144, 32000 ], "size": 6684672, "type": 14 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 71333265408, "shape": [ 6144, 16384, 8 ], "size": 161280000, "type": 11 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 71679295488, "shape": [ 16384, 6144, 8 ], "size": 346030080, "type": 13 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 72232943616, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 11 }, { "name": "blk.55.attn_norm.weight", "offset": 72578973696, "shape": [ 6144 ], "size": 346030080, "type": 0 }, { "name": "blk.55.ffn_norm.weight", "offset": 72578998272, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "output_norm.weight", "offset": 72579022848, "shape": [ 6144 ], "size": 24576, "type": 0 } ], "version": 3 }