dolphin-mixtral

Uncensored, 8x7b and 8x22b fine-tuned models based on the Mixtral mixture of experts models that excels at coding tasks. Created by Eric Hartford.

8x7B 8x22B

276.6K Pulls Updated 6 weeks ago

87 Tags

... /
model
baff6030550d · 100GB
{ "metadata": { "general.architecture": "llama", "general.file_type": 17, "general.name": "dolphin-2.9-mixtral-8x22b", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 65536, "llama.embedding_length": 6144, "llama.expert_count": 8, "llama.expert_used_count": 2, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32002, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32002 values)", "tokenizer.ggml.token_type": "... (32002 values)", "tokenizer.ggml.tokens": "... (32002 values)" }, "num_params": 140620658688, "tensors": [ { "name": "blk.0.ffn_gate_exps.weight", "offset": 135176448, "shape": [ 6144, 16384, 8 ], "size": 135176448, "type": 13 }, { "name": "blk.0.ffn_gate_inp.weight", "offset": 688824576, "shape": [ 6144, 8 ], "size": 553648128, "type": 1 }, { "name": "blk.0.attn_k.weight", "offset": 688922880, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 695607552, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 721559808, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.0.attn_v.weight", "offset": 747512064, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.0.ffn_down_exps.weight", "offset": 754196736, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 14 }, { "name": "blk.0.ffn_up_exps.weight", "offset": 1414799616, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 1968447744, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 1968472320, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1968496896, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.1.attn_k.weight", "offset": 1968595200, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 1975279872, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 2001232128, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.1.attn_v.weight", "offset": 2027184384, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 2033869056, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 2587517184, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 3248120064, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 3801768192, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 3801792768, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 3801817344, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.2.attn_k.weight", "offset": 3801915648, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 3808600320, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 3834552576, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.2.attn_v.weight", "offset": 3860504832, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 3867189504, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 4420837632, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 5081440512, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 5635088640, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 5635113216, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 5635137792, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.3.attn_k.weight", "offset": 5635236096, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 5641920768, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 5667873024, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.3.attn_v.weight", "offset": 5693825280, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 5700509952, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 6254158080, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 6914760960, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 7468409088, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 7468433664, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 7468458240, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.4.attn_k.weight", "offset": 7468556544, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 7475241216, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 7501193472, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.4.attn_v.weight", "offset": 7527145728, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 7533830400, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 8087478528, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 8748081408, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 9301729536, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 9301754112, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 9301778688, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.5.attn_k.weight", "offset": 9301876992, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 9308561664, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 9334513920, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.5.attn_v.weight", "offset": 9360466176, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 9367150848, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 9920798976, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 10581401856, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 11135049984, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 11135074560, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 11135099136, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.6.attn_k.weight", "offset": 11135197440, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 11141882112, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 11167834368, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.6.attn_v.weight", "offset": 11193786624, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 11200471296, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 11754119424, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 12414722304, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 12968370432, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 12968395008, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 12968419584, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.7.attn_k.weight", "offset": 12968517888, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 12975202560, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 13001154816, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.7.attn_v.weight", "offset": 13027107072, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 13033791744, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 13587439872, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 14141088000, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 14694736128, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 14694760704, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 14694785280, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.8.attn_k.weight", "offset": 14694883584, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 14701568256, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 14727520512, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.8.attn_v.weight", "offset": 14753472768, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 14760157440, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 15313805568, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 15867453696, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 16421101824, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 16421126400, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 16421150976, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.9.attn_k.weight", "offset": 16421249280, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 16427933952, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 16453886208, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.9.attn_v.weight", "offset": 16479838464, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 16486523136, "shape": [ 6144, 8 ], "size": 6684672, "type": 1 }, { "name": "blk.10.attn_k.weight", "offset": 16486621440, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 16493306112, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 16519258368, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.10.attn_v.weight", "offset": 16545210624, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 16551895296, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 17105543424, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 17766146304, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 18319794432, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 18319819008, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 18319843584, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 13 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 18873491712, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "\u0000\u0000\u0018\u0000\u0000\u0000\u0000\u0000\u0000\b\u0000\u0000\u0000\u0000\u0000\u0000\u0000\r\u0000\u0000\u0000\u0000a\ufffd\ufffd", "offset": 19427139840, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 19980787968, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 19980812544, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 19980837120, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.11.attn_k.weight", "offset": 19980935424, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 19987620096, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 20013572352, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.11.attn_v.weight", "offset": 20039524608, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 20046209280, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 20599857408, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 21153505536, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 21707153664, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 21707178240, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 21707202816, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.12.attn_k.weight", "offset": 21707301120, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 21713985792, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 21739938048, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.12.attn_v.weight", "offset": 21765890304, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 21772574976, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 22326223104, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 22986825984, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 23540474112, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 23540498688, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 23540523264, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.13.attn_k.weight", "offset": 23540621568, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 23547306240, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 23573258496, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.13.attn_v.weight", "offset": 23599210752, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 23605895424, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 24159543552, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 24713191680, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 25266839808, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 25266864384, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 25266888960, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.14.attn_k.weight", "offset": 25266987264, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 25273671936, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 25299624192, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.14.attn_v.weight", "offset": 25325576448, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 25332261120, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 25885909248, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 26439557376, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 26993205504, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 26993230080, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 26993254656, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.15.attn_k.weight", "offset": 26993352960, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 27000037632, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 27025989888, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.15.attn_v.weight", "offset": 27051942144, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 27058626816, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 27612274944, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 28272877824, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 28826525952, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 28826550528, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 28826575104, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.16.attn_k.weight", "offset": 28826673408, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 28833358080, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 28859310336, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.16.attn_v.weight", "offset": 28885262592, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 28891947264, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 29445595392, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 29999243520, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 30552891648, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 30552916224, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 30552940800, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.17.attn_k.weight", "offset": 30553039104, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 30559723776, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 30585676032, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.17.attn_v.weight", "offset": 30611628288, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 30618312960, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 31171961088, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 31725609216, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 32279257344, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 32279281920, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 32279306496, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.18.attn_k.weight", "offset": 32279404800, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 32286089472, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 32312041728, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.18.attn_v.weight", "offset": 32337993984, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 32344678656, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 32898326784, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 33558929664, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 34112577792, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 34112602368, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 34112626944, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.19.attn_k.weight", "offset": 34112725248, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 34119409920, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 34145362176, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.19.attn_v.weight", "offset": 34171314432, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 34177999104, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 34731647232, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 35285295360, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 35838943488, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 35838968064, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 35838992640, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.20.attn_k.weight", "offset": 35839090944, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 35845775616, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 35871727872, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.20.attn_v.weight", "offset": 35897680128, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 35904364800, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 36458012928, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 37011661056, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 37565309184, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 37565333760, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 37565358336, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.21.attn_k.weight", "offset": 37565456640, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 37572141312, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 37598093568, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.21.attn_v.weight", "offset": 37624045824, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 37630730496, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 38184378624, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 38844981504, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 39398629632, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 39398654208, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 39398678784, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.22.attn_k.weight", "offset": 39398777088, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 39405461760, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 39431414016, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.22.attn_v.weight", "offset": 39457366272, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 39464050944, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 40017699072, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 40571347200, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 41124995328, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 41125019904, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 41125044480, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.23.attn_q.weight", "offset": 41131729152, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.23.attn_v.weight", "offset": 41157681408, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 41164366080, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 41718014208, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 42271662336, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 42825310464, "shape": [ 6144, 8 ], "size": 553648128, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 42825408768, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 42825433344, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 42825457920, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.24.attn_k.weight", "offset": 42851410176, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.24.attn_q.weight", "offset": 42858094848, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 42884047104, "shape": [ 6144, 16384, 8 ], "size": 25952256, "type": 13 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 43437695232, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 44098298112, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 44651946240, "shape": [ 6144, 8 ], "size": 553648128, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 44652044544, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 44652069120, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 44652093696, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.24.attn_v.weight", "offset": 44678045952, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.25.attn_q.weight", "offset": 44684730624, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 44710682880, "shape": [ 6144, 16384, 8 ], "size": 25952256, "type": 13 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 45264331008, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 45817979136, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 46371627264, "shape": [ 6144, 8 ], "size": 553648128, "type": 1 }, { "name": "blk.25.attn_norm.weight", "offset": 46371725568, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 46371750144, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 46371774720, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 46378459392, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.25.attn_v.weight", "offset": 46404411648, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 46411096320, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 46964744448, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 47518392576, "shape": [ 6144, 8 ], "size": 553648128, "type": 1 }, { "name": "blk.26.attn_k.weight", "offset": 47518490880, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 47525175552, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 47551127808, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.26.attn_v.weight", "offset": 47577080064, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 47583764736, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 48137412864, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 48137437440, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 48137462016, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 13 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 48691110144, "shape": [ 6144, 8 ], "size": 553648128, "type": 1 }, { "name": "blk.27.attn_k.weight", "offset": 48691208448, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 48697893120, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.27.attn_q.weight", "offset": 48723845376, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.27.attn_v.weight", "offset": 48749797632, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 48756482304, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 14 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 49417085184, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 49970733312, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 49970757888, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 49970782464, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.28.attn_k.weight", "offset": 49970880768, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 49977565440, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 50003517696, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.28.attn_v.weight", "offset": 50029469952, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 50036154624, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 50589802752, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 51143450880, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 51697099008, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 51697123584, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 51697148160, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.29.attn_k.weight", "offset": 51697246464, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 51703931136, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 51729883392, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.29.attn_v.weight", "offset": 51755835648, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 51762520320, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 52316168448, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 52869816576, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 53423464704, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 53423489280, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 53423513856, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.30.attn_k.weight", "offset": 53423612160, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 53430296832, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 53456249088, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.30.attn_v.weight", "offset": 53482201344, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 53488886016, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 54042534144, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 54703137024, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 55256785152, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 55256809728, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 55256834304, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.31.attn_k.weight", "offset": 55256932608, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 55263617280, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.31.attn_q.weight", "offset": 55289569536, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.31.attn_v.weight", "offset": 55315521792, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 55322206464, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 55875854592, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 56429502720, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 56983150848, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 56983175424, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 56983200000, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.32.attn_k.weight", "offset": 56983298304, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.32.attn_output.weight", "offset": 56989982976, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.32.attn_q.weight", "offset": 57015935232, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.32.attn_v.weight", "offset": 57041887488, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 57048572160, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 57602220288, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 58155868416, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 58709516544, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 58709541120, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 58709565696, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.33.attn_k.weight", "offset": 58709664000, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.33.attn_output.weight", "offset": 58716348672, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.33.attn_q.weight", "offset": 58742300928, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.33.attn_v.weight", "offset": 58768253184, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 58774937856, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 59328585984, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 59989188864, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.33.attn_norm.weight", "offset": 60542836992, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 60542861568, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 60542886144, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.34.attn_k.weight", "offset": 60542984448, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.34.attn_output.weight", "offset": 60549669120, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.34.attn_q.weight", "offset": 60575621376, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.34.attn_v.weight", "offset": 60601573632, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 60608258304, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 61161906432, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 61715554560, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 62269202688, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 62269227264, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 62269251840, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.35.attn_k.weight", "offset": 62269350144, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.35.attn_output.weight", "offset": 62276034816, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.35.attn_q.weight", "offset": 62301987072, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.35.attn_v.weight", "offset": 62327939328, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 62334624000, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 62888272128, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 63441920256, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.35.attn_norm.weight", "offset": 63995568384, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 63995592960, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 63995617536, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.36.attn_k.weight", "offset": 63995715840, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "rm.weight\u0001\u0000\u0000\u0000\u0000\u0018\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", "offset": 64002400512, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.36.attn_q.weight", "offset": 64028352768, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.36.attn_v.weight", "offset": 64054305024, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 64060989696, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 64614637824, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 65275240704, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 65828888832, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 65828913408, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 65828937984, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.37.attn_k.weight", "offset": 65829036288, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.37.attn_output.weight", "offset": 65835720960, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.37.attn_q.weight", "offset": 65861673216, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.37.attn_v.weight", "offset": 65887625472, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 65894310144, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 66447958272, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 67001606400, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.37.attn_norm.weight", "offset": 67555254528, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 67555279104, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 67555303680, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.38.attn_k.weight", "offset": 67555401984, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.38.attn_output.weight", "offset": 67562086656, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.38.attn_q.weight", "offset": 67588038912, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.38.attn_v.weight", "offset": 67613991168, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 67620675840, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 68174323968, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 68727972096, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 69281620224, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 69281644800, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 69281669376, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.39.attn_k.weight", "offset": 69281767680, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.39.attn_output.weight", "offset": 69288452352, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.39.attn_q.weight", "offset": 69314404608, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.39.attn_v.weight", "offset": 69340356864, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 69347041536, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 69900689664, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 70561292544, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 71114940672, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 71114965248, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 71114989824, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.40.attn_k.weight", "offset": 71115088128, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.40.attn_output.weight", "offset": 71121772800, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.40.attn_q.weight", "offset": 71147725056, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.40.attn_v.weight", "offset": 71173677312, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 71180361984, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 71734010112, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 72287658240, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 72841306368, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 72841330944, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 72841355520, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.41.attn_k.weight", "offset": 72841453824, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.41.attn_output.weight", "offset": 72848138496, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.41.attn_q.weight", "offset": 72874090752, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.41.attn_v.weight", "offset": 72900043008, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 72906727680, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 73460375808, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 74014023936, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 74567672064, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 74567696640, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 74567721216, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.42.attn_k.weight", "offset": 74567819520, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.42.attn_output.weight", "offset": 74574504192, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.42.attn_q.weight", "offset": 74600456448, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.42.attn_v.weight", "offset": 74626408704, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 74633093376, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 75186741504, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 75847344384, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.42.attn_norm.weight", "offset": 76400992512, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 76401017088, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 76401041664, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.43.attn_k.weight", "offset": 76401139968, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.43.attn_output.weight", "offset": 76407824640, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.43.attn_q.weight", "offset": 76433776896, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.43.attn_v.weight", "offset": 76459729152, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 76466413824, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 77020061952, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 77573710080, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 78127358208, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 78127382784, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 78127407360, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.44.attn_k.weight", "offset": 78127505664, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.44.attn_output.weight", "offset": 78134190336, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.44.attn_q.weight", "offset": 78160142592, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.44.attn_v.weight", "offset": 78186094848, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 78192779520, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 78746427648, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 79300075776, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.44.attn_norm.weight", "offset": 79853723904, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 79853748480, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 79853773056, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.45.attn_k.weight", "offset": 79853871360, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.45.attn_output.weight", "offset": 79860556032, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.45.attn_q.weight", "offset": 79886508288, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.45.attn_v.weight", "offset": 79912460544, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 79919145216, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 80472793344, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 81133396224, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 81687044352, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 81687068928, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 81687093504, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.46.attn_k.weight", "offset": 81687191808, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.46.attn_output.weight", "offset": 81693876480, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.46.attn_q.weight", "offset": 81719828736, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.46.attn_v.weight", "offset": 81745780992, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 81752465664, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 82306113792, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 82859761920, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.46.attn_norm.weight", "offset": 83413410048, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.46.ffn_norm.weight", "offset": 83413434624, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 83413459200, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.47.attn_k.weight", "offset": 83413557504, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.47.attn_output.weight", "offset": 83420242176, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.47.attn_q.weight", "offset": 83446194432, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.47.attn_v.weight", "offset": 83472146688, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 83478831360, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 84032479488, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 84586127616, "shape": [ 6144, 16384, 8 ], "size": 553648128, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 85139775744, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.47.ffn_norm.weight", "offset": 85139800320, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 85139824896, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.48.attn_k.weight", "offset": 85139923200, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.48.attn_output.weight", "offset": 85146607872, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.48.attn_q.weight", "offset": 85172560128, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.48.attn_v.weight", "offset": 85198512384, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 85205197056, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 85758845184, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 86419448064, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.48.attn_norm.weight", "offset": 86973096192, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.48.ffn_norm.weight", "offset": 86973120768, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 86973145344, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.49.attn_k.weight", "offset": 86973243648, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.49.attn_output.weight", "offset": 86979928320, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.49.attn_q.weight", "offset": 87005880576, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.49.attn_v.weight", "offset": 87031832832, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 87038517504, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 87592165632, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 88252768512, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.49.attn_norm.weight", "offset": 88806416640, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.49.ffn_norm.weight", "offset": 88806441216, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 88806465792, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.50.attn_q.weight", "offset": 88813150464, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.50.attn_v.weight", "offset": 88839102720, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 88845787392, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 89399435520, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 90060038400, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 90613686528, "shape": [ 6144, 8 ], "size": 553648128, "type": 1 }, { "name": "blk.50.attn_norm.weight", "offset": 90613784832, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.50.ffn_norm.weight", "offset": 90613809408, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 90613833984, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.51.attn_k.weight", "offset": 90639786240, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.51.attn_q.weight", "offset": 90646470912, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 90672423168, "shape": [ 6144, 16384, 8 ], "size": 25952256, "type": 13 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 91226071296, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 91886674176, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 92440322304, "shape": [ 6144, 8 ], "size": 553648128, "type": 1 }, { "name": "blk.51.attn_norm.weight", "offset": 92440420608, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.51.ffn_norm.weight", "offset": 92440445184, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.51.attn_output.weight", "offset": 92440469760, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.51.attn_v.weight", "offset": 92466422016, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.52.attn_q.weight", "offset": 92473106688, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 92499058944, "shape": [ 6144, 16384, 8 ], "size": 25952256, "type": 13 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 93052707072, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 93713309952, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 94266958080, "shape": [ 6144, 8 ], "size": 553648128, "type": 1 }, { "name": "blk.52.attn_norm.weight", "offset": 94267056384, "shape": [ 6144 ], "size": 98304, "type": 0 }, { "name": "blk.52.ffn_norm.weight", "offset": 94267080960, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 94267105536, "shape": [ 6144, 1024 ], "size": 24576, "type": 8 }, { "name": "blk.52.attn_output.weight", "offset": 94273790208, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.52.attn_v.weight", "offset": 94299742464, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 94306427136, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 94860075264, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 95520678144, "shape": [ 6144, 8 ], "size": 660602880, "type": 1 }, { "name": "blk.53.attn_k.weight", "offset": 95520776448, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.53.attn_output.weight", "offset": 95527461120, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.53.attn_q.weight", "offset": 95553413376, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.53.attn_v.weight", "offset": 95579365632, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 95586050304, "shape": [ 6144, 16384, 8 ], "size": 6684672, "type": 13 }, { "name": "blk.53.attn_norm.weight", "offset": 96139698432, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.53.ffn_norm.weight", "offset": 96139723008, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 96139747584, "shape": [ 6144, 16384, 8 ], "size": 24576, "type": 13 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 96693395712, "shape": [ 6144, 8 ], "size": 553648128, "type": 1 }, { "name": "blk.54.attn_k.weight", "offset": 96693494016, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.54.attn_output.weight", "offset": 96700178688, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.54.attn_q.weight", "offset": 96726130944, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.54.attn_v.weight", "offset": 96752083200, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 96758767872, "shape": [ 16384, 6144, 8 ], "size": 6684672, "type": 14 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 97419370752, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.54.attn_norm.weight", "offset": 97973018880, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.54.ffn_norm.weight", "offset": 97973043456, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 97973068032, "shape": [ 6144, 8 ], "size": 24576, "type": 1 }, { "name": "blk.55.attn_k.weight", "offset": 97973166336, "shape": [ 6144, 1024 ], "size": 98304, "type": 8 }, { "name": "blk.55.attn_output.weight", "offset": 97979851008, "shape": [ 6144, 6144 ], "size": 6684672, "type": 13 }, { "name": "blk.55.attn_q.weight", "offset": 98005803264, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.55.attn_v.weight", "offset": 98031755520, "shape": [ 6144, 1024 ], "size": 25952256, "type": 8 }, { "name": "output.weight", "offset": 98038440192, "shape": [ 6144, 32002 ], "size": 6684672, "type": 14 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 98199730272, "shape": [ 6144, 16384, 8 ], "size": 161290080, "type": 13 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 98753378400, "shape": [ 16384, 6144, 8 ], "size": 553648128, "type": 14 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 99413981280, "shape": [ 6144, 16384, 8 ], "size": 660602880, "type": 13 }, { "name": "blk.55.attn_norm.weight", "offset": 99967629408, "shape": [ 6144 ], "size": 553648128, "type": 0 }, { "name": "blk.55.ffn_norm.weight", "offset": 99967653984, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "output_norm.weight", "offset": 99967678560, "shape": [ 6144 ], "size": 24576, "type": 0 } ], "version": 3 }