yi

{ "metadata": { "general.architecture": "llama", "general.file_type": 1, "general.name": "Yi-1.5-34B-Chat", "llama.attention.head_count": 56, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 60, "llama.context_length": 4096, "llama.embedding_length": 7168, "llama.feed_forward_length": 20480, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 5000000, "llama.vocab_size": 64000, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (64000 values)", "tokenizer.ggml.token_type": "... (64000 values)", "tokenizer.ggml.tokens": "... (64000 values)" }, "num_params": 34388917248, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 917504000, "shape": [ 7168 ], "size": 917504000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 917532672, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.0.ffn_gate.weight", "offset": 1211133952, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.0.ffn_up.weight", "offset": 1504735232, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.0.ffn_norm.weight", "offset": 1798336512, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 1798365184, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.0.attn_output.weight", "offset": 1813045248, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.0.attn_q.weight", "offset": 1915805696, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.0.attn_v.weight", "offset": 2018566144, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.1.attn_norm.weight", "offset": 2033246208, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 2033274880, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.1.ffn_gate.weight", "offset": 2326876160, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.1.ffn_up.weight", "offset": 2620477440, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.1.ffn_norm.weight", "offset": 2914078720, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 2914107392, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.1.attn_output.weight", "offset": 2928787456, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.1.attn_q.weight", "offset": 3031547904, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.1.attn_v.weight", "offset": 3134308352, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 3148988416, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 3149017088, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.2.ffn_gate.weight", "offset": 3442618368, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.2.ffn_up.weight", "offset": 3736219648, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.2.ffn_norm.weight", "offset": 4029820928, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 4029849600, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.2.attn_output.weight", "offset": 4044529664, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.2.attn_q.weight", "offset": 4147290112, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.2.attn_v.weight", "offset": 4250050560, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.3.ffn_gate.weight", "offset": 4264730624, "shape": [ 7168, 20480 ], "size": 14680064, "type": 1 }, { "name": "blk.3.attn_k.weight", "offset": 4558331904, "shape": [ 7168, 1024 ], "size": 293601280, "type": 1 }, { "name": "blk.3.attn_output.weight", "offset": 4573011968, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.3.attn_q.weight", "offset": 4675772416, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.3.attn_v.weight", "offset": 4778532864, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.3.attn_norm.weight", "offset": 4793212928, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 4793241600, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.3.ffn_up.weight", "offset": 5086842880, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.3.ffn_norm.weight", "offset": 5380444160, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 5380472832, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 5380501504, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.4.ffn_gate.weight", "offset": 5674102784, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.4.ffn_up.weight", "offset": 5967704064, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.4.ffn_norm.weight", "offset": 6261305344, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 6261334016, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.4.attn_output.weight", "offset": 6276014080, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.4.attn_q.weight", "offset": 6378774528, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.4.attn_v.weight", "offset": 6481534976, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.5.attn_norm.weight", "offset": 6496215040, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 6496243712, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.5.ffn_gate.weight", "offset": 6789844992, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.5.ffn_up.weight", "offset": 7083446272, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.5.ffn_norm.weight", "offset": 7377047552, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 7377076224, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.5.attn_output.weight", "offset": 7391756288, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.5.attn_q.weight", "offset": 7494516736, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.5.attn_v.weight", "offset": 7597277184, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.6.attn_norm.weight", "offset": 7611957248, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 7611985920, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.6.ffn_gate.weight", "offset": 7905587200, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.6.ffn_up.weight", "offset": 8199188480, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.6.ffn_norm.weight", "offset": 8492789760, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 8492818432, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.6.attn_output.weight", "offset": 8507498496, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.6.attn_q.weight", "offset": 8610258944, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.6.attn_v.weight", "offset": 8713019392, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.7.ffn_gate.weight", "offset": 8727699456, "shape": [ 7168, 20480 ], "size": 14680064, "type": 1 }, { "name": "blk.7.ffn_up.weight", "offset": 9021300736, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.7.attn_k.weight", "offset": 9314902016, "shape": [ 7168, 1024 ], "size": 293601280, "type": 1 }, { "name": "blk.7.attn_output.weight", "offset": 9329582080, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.7.attn_q.weight", "offset": 9432342528, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.7.attn_v.weight", "offset": 9535102976, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.10.attn_norm.weight", "offset": 9549783040, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 9549811712, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.10.ffn_gate.weight", "offset": 9843412992, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.10.ffn_up.weight", "offset": 10137014272, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.10.ffn_norm.weight", "offset": 10430615552, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 10430644224, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.10.attn_output.weight", "offset": 10445324288, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.10.attn_q.weight", "offset": 10548084736, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.10.attn_v.weight", "offset": 10650845184, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.11.attn_norm.weight", "offset": 10665525248, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 10665553920, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.11.ffn_gate.weight", "offset": 10959155200, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.11.ffn_up.weight", "offset": 11252756480, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.11.ffn_norm.weight", "offset": 11546357760, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 11546386432, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.11.attn_output.weight", "offset": 11561066496, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.11.attn_q.weight", "offset": 11663826944, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.11.attn_v.weight", "offset": 11766587392, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.12.attn_k.weight", "offset": 11781267456, "shape": [ 7168, 1024 ], "size": 14680064, "type": 1 }, { "name": "blk.12.attn_output.weight", "offset": 11795947520, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.12.attn_q.weight", "offset": 11898707968, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.12.attn_v.weight", "offset": 12001468416, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.7.attn_norm.weight", "offset": 12016148480, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 12016177152, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.7.ffn_norm.weight", "offset": 12309778432, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 12309807104, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 12309835776, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.8.ffn_gate.weight", "offset": 12603437056, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.8.ffn_up.weight", "offset": 12897038336, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.8.ffn_norm.weight", "offset": 13190639616, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 13190668288, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.8.attn_output.weight", "offset": 13205348352, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.8.attn_q.weight", "offset": 13308108800, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.8.attn_v.weight", "offset": 13410869248, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 13425549312, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 13425577984, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.9.ffn_gate.weight", "offset": 13719179264, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.9.ffn_up.weight", "offset": 14012780544, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.9.ffn_norm.weight", "offset": 14306381824, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 14306410496, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.9.attn_output.weight", "offset": 14321090560, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.9.attn_q.weight", "offset": 14423851008, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.9.attn_v.weight", "offset": 14526611456, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 14541291520, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 14541320192, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.12.ffn_gate.weight", "offset": 14834921472, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.12.ffn_up.weight", "offset": 15128522752, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.12.ffn_norm.weight", "offset": 15422124032, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 15422152704, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 15422181376, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.13.ffn_gate.weight", "offset": 15715782656, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.13.ffn_up.weight", "offset": 16009383936, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.13.ffn_norm.weight", "offset": 16302985216, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 16303013888, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.13.attn_output.weight", "offset": 16317693952, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.13.attn_q.weight", "offset": 16420454400, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.13.attn_v.weight", "offset": 16523214848, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 16537894912, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 16537923584, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.14.ffn_gate.weight", "offset": 16831524864, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.14.ffn_up.weight", "offset": 17125126144, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.14.ffn_norm.weight", "offset": 17418727424, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 17418756096, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.14.attn_output.weight", "offset": 17433436160, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.14.attn_q.weight", "offset": 17536196608, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.14.attn_v.weight", "offset": 17638957056, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.15.attn_norm.weight", "offset": 17653637120, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 17653665792, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.15.ffn_gate.weight", "offset": 17947267072, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.15.ffn_up.weight", "offset": 18240868352, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.15.ffn_norm.weight", "offset": 18534469632, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 18534498304, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.15.attn_output.weight", "offset": 18549178368, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.15.attn_q.weight", "offset": 18651938816, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.15.attn_v.weight", "offset": 18754699264, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.16.ffn_gate.weight", "offset": 18769379328, "shape": [ 7168, 20480 ], "size": 14680064, "type": 1 }, { "name": "blk.16.attn_k.weight", "offset": 19062980608, "shape": [ 7168, 1024 ], "size": 293601280, "type": 1 }, { "name": "blk.16.attn_output.weight", "offset": 19077660672, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.16.attn_q.weight", "offset": 19180421120, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.16.attn_v.weight", "offset": 19283181568, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.16.attn_norm.weight", "offset": 19297861632, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 19297890304, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.16.ffn_up.weight", "offset": 19591491584, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.16.ffn_norm.weight", "offset": 19885092864, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 19885121536, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 19885150208, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.17.ffn_gate.weight", "offset": 20178751488, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.17.ffn_up.weight", "offset": 20472352768, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.17.ffn_norm.weight", "offset": 20765954048, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 20765982720, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.17.attn_output.weight", "offset": 20780662784, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.17.attn_q.weight", "offset": 20883423232, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.17.attn_v.weight", "offset": 20986183680, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.18.attn_norm.weight", "offset": 21000863744, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 21000892416, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.18.ffn_gate.weight", "offset": 21294493696, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.18.ffn_up.weight", "offset": 21588094976, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.18.ffn_norm.weight", "offset": 21881696256, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 21881724928, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.18.attn_output.weight", "offset": 21896404992, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.18.attn_q.weight", "offset": 21999165440, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.18.attn_v.weight", "offset": 22101925888, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.19.attn_norm.weight", "offset": 22116605952, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 22116634624, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.19.ffn_gate.weight", "offset": 22410235904, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.19.ffn_up.weight", "offset": 22703837184, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.19.ffn_norm.weight", "offset": 22997438464, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 22997467136, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.19.attn_output.weight", "offset": 23012147200, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.19.attn_q.weight", "offset": 23114907648, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.19.attn_v.weight", "offset": 23217668096, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.20.ffn_gate.weight", "offset": 23232348160, "shape": [ 7168, 20480 ], "size": 14680064, "type": 1 }, { "name": "blk.20.ffn_up.weight", "offset": 23525949440, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.20.attn_k.weight", "offset": 23819550720, "shape": [ 7168, 1024 ], "size": 293601280, "type": 1 }, { "name": "blk.20.attn_output.weight", "offset": 23834230784, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.20.attn_q.weight", "offset": 23936991232, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.20.attn_v.weight", "offset": 24039751680, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.20.attn_norm.weight", "offset": 24054431744, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 24054460416, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.20.ffn_norm.weight", "offset": 24348061696, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.21.attn_norm.weight", "offset": 24348090368, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 24348119040, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.21.ffn_gate.weight", "offset": 24641720320, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.21.ffn_up.weight", "offset": 24935321600, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.21.ffn_norm.weight", "offset": 25228922880, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 25228951552, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.21.attn_output.weight", "offset": 25243631616, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.21.attn_q.weight", "offset": 25346392064, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.21.attn_v.weight", "offset": 25449152512, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 25463832576, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 25463861248, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.22.ffn_gate.weight", "offset": 25757462528, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.22.ffn_up.weight", "offset": 26051063808, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.22.ffn_norm.weight", "offset": 26344665088, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 26344693760, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.22.attn_output.weight", "offset": 26359373824, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.22.attn_q.weight", "offset": 26462134272, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.22.attn_v.weight", "offset": 26564894720, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 26579574784, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 26579603456, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.23.ffn_gate.weight", "offset": 26873204736, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.23.ffn_up.weight", "offset": 27166806016, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.23.ffn_norm.weight", "offset": 27460407296, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 27460435968, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.23.attn_output.weight", "offset": 27475116032, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.23.attn_q.weight", "offset": 27577876480, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.23.attn_v.weight", "offset": 27680636928, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 27695316992, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 27695345664, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.24.ffn_gate.weight", "offset": 27988946944, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.24.ffn_up.weight", "offset": 28282548224, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.24.ffn_norm.weight", "offset": 28576149504, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 28576178176, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.24.attn_output.weight", "offset": 28590858240, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.24.attn_q.weight", "offset": 28693618688, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.24.attn_v.weight", "offset": 28796379136, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.25.attn_k.weight", "offset": 28811059200, "shape": [ 7168, 1024 ], "size": 14680064, "type": 1 }, { "name": "blk.25.attn_output.weight", "offset": 28825739264, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.25.attn_q.weight", "offset": 28928499712, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.25.attn_v.weight", "offset": 29031260160, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.25.attn_norm.weight", "offset": 29045940224, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 29045968896, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.25.ffn_gate.weight", "offset": 29339570176, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.25.ffn_up.weight", "offset": 29633171456, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.25.ffn_norm.weight", "offset": 29926772736, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 29926801408, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 29926830080, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.26.ffn_gate.weight", "offset": 30220431360, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.26.ffn_up.weight", "offset": 30514032640, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.26.ffn_norm.weight", "offset": 30807633920, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 30807662592, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.26.attn_output.weight", "offset": 30822342656, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.26.attn_q.weight", "offset": 30925103104, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.26.attn_v.weight", "offset": 31027863552, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.27.attn_norm.weight", "offset": 31042543616, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 31042572288, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.27.ffn_gate.weight", "offset": 31336173568, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.27.ffn_up.weight", "offset": 31629774848, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.27.ffn_norm.weight", "offset": 31923376128, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 31923404800, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.27.attn_output.weight", "offset": 31938084864, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.27.attn_q.weight", "offset": 32040845312, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.27.attn_v.weight", "offset": 32143605760, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.28.attn_norm.weight", "offset": 32158285824, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 32158314496, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.28.ffn_gate.weight", "offset": 32451915776, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.28.ffn_up.weight", "offset": 32745517056, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.28.ffn_norm.weight", "offset": 33039118336, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 33039147008, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.28.attn_output.weight", "offset": 33053827072, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.28.attn_q.weight", "offset": 33156587520, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.28.attn_v.weight", "offset": 33259347968, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.29.ffn_gate.weight", "offset": 33274028032, "shape": [ 7168, 20480 ], "size": 14680064, "type": 1 }, { "name": "blk.29.attn_k.weight", "offset": 33567629312, "shape": [ 7168, 1024 ], "size": 293601280, "type": 1 }, { "name": "blk.29.attn_output.weight", "offset": 33582309376, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.29.attn_q.weight", "offset": 33685069824, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.29.attn_v.weight", "offset": 33787830272, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.29.attn_norm.weight", "offset": 33802510336, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 33802539008, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.29.ffn_up.weight", "offset": 34096140288, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.29.ffn_norm.weight", "offset": 34389741568, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.30.attn_norm.weight", "offset": 34389770240, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 34389798912, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.30.ffn_gate.weight", "offset": 34683400192, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.30.ffn_up.weight", "offset": 34977001472, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.30.ffn_norm.weight", "offset": 35270602752, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 35270631424, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.30.attn_output.weight", "offset": 35285311488, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.30.attn_q.weight", "offset": 35388071936, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.30.attn_v.weight", "offset": 35490832384, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.31.attn_norm.weight", "offset": 35505512448, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 35505541120, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.31.ffn_gate.weight", "offset": 35799142400, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.31.ffn_up.weight", "offset": 36092743680, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.31.ffn_norm.weight", "offset": 36386344960, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 36386373632, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.31.attn_output.weight", "offset": 36401053696, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.31.attn_q.weight", "offset": 36503814144, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.31.attn_v.weight", "offset": 36606574592, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.32.attn_norm.weight", "offset": 36621254656, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 36621283328, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.32.ffn_gate.weight", "offset": 36914884608, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.32.ffn_up.weight", "offset": 37208485888, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.32.ffn_norm.weight", "offset": 37502087168, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 37502115840, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.32.attn_output.weight", "offset": 37516795904, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.32.attn_q.weight", "offset": 37619556352, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.32.attn_v.weight", "offset": 37722316800, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.33.ffn_gate.weight", "offset": 37736996864, "shape": [ 7168, 20480 ], "size": 14680064, "type": 1 }, { "name": "blk.33.ffn_up.weight", "offset": 38030598144, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.33.attn_k.weight", "offset": 38324199424, "shape": [ 7168, 1024 ], "size": 293601280, "type": 1 }, { "name": "blk.33.attn_output.weight", "offset": 38338879488, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.33.attn_q.weight", "offset": 38441639936, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.33.attn_v.weight", "offset": 38544400384, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.33.attn_norm.weight", "offset": 38559080448, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 38559109120, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.33.ffn_norm.weight", "offset": 38852710400, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.34.attn_norm.weight", "offset": 38852739072, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 38852767744, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.34.ffn_gate.weight", "offset": 39146369024, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.34.ffn_up.weight", "offset": 39439970304, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.34.ffn_norm.weight", "offset": 39733571584, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 39733600256, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.34.attn_output.weight", "offset": 39748280320, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.34.attn_q.weight", "offset": 39851040768, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.34.attn_v.weight", "offset": 39953801216, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.35.attn_norm.weight", "offset": 39968481280, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 39968509952, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.35.ffn_gate.weight", "offset": 40262111232, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.35.ffn_up.weight", "offset": 40555712512, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.35.ffn_norm.weight", "offset": 40849313792, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 40849342464, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.35.attn_output.weight", "offset": 40864022528, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.35.attn_q.weight", "offset": 40966782976, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.35.attn_v.weight", "offset": 41069543424, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.36.attn_norm.weight", "offset": 41084223488, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 41084252160, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.36.ffn_gate.weight", "offset": 41377853440, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.36.ffn_up.weight", "offset": 41671454720, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.36.ffn_norm.weight", "offset": 41965056000, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 41965084672, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.36.attn_output.weight", "offset": 41979764736, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.36.attn_q.weight", "offset": 42082525184, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.36.attn_v.weight", "offset": 42185285632, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.37.attn_norm.weight", "offset": 42199965696, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 42199994368, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.37.ffn_gate.weight", "offset": 42493595648, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.37.ffn_up.weight", "offset": 42787196928, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.37.ffn_norm.weight", "offset": 43080798208, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 43080826880, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.37.attn_output.weight", "offset": 43095506944, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.37.attn_q.weight", "offset": 43198267392, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.37.attn_v.weight", "offset": 43301027840, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.38.attn_k.weight", "offset": 43315707904, "shape": [ 7168, 1024 ], "size": 14680064, "type": 1 }, { "name": "blk.38.attn_output.weight", "offset": 43330387968, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.38.attn_q.weight", "offset": 43433148416, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.38.attn_v.weight", "offset": 43535908864, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.38.attn_norm.weight", "offset": 43550588928, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 43550617600, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.38.ffn_gate.weight", "offset": 43844218880, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.38.ffn_up.weight", "offset": 44137820160, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.38.ffn_norm.weight", "offset": 44431421440, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.39.attn_norm.weight", "offset": 44431450112, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 44431478784, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.39.ffn_gate.weight", "offset": 44725080064, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.39.ffn_up.weight", "offset": 45018681344, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.39.ffn_norm.weight", "offset": 45312282624, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 45312311296, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.39.attn_output.weight", "offset": 45326991360, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.39.attn_q.weight", "offset": 45429751808, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.39.attn_v.weight", "offset": 45532512256, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.40.attn_norm.weight", "offset": 45547192320, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 45547220992, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.40.ffn_gate.weight", "offset": 45840822272, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.40.ffn_up.weight", "offset": 46134423552, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.40.ffn_norm.weight", "offset": 46428024832, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 46428053504, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.40.attn_output.weight", "offset": 46442733568, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.40.attn_q.weight", "offset": 46545494016, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.40.attn_v.weight", "offset": 46648254464, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.41.attn_norm.weight", "offset": 46662934528, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 46662963200, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.41.ffn_gate.weight", "offset": 46956564480, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.41.ffn_up.weight", "offset": 47250165760, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.41.ffn_norm.weight", "offset": 47543767040, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 47543795712, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.41.attn_output.weight", "offset": 47558475776, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.41.attn_q.weight", "offset": 47661236224, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.41.attn_v.weight", "offset": 47763996672, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.42.ffn_gate.weight", "offset": 47778676736, "shape": [ 7168, 20480 ], "size": 14680064, "type": 1 }, { "name": "blk.42.attn_k.weight", "offset": 48072278016, "shape": [ 7168, 1024 ], "size": 293601280, "type": 1 }, { "name": "blk.42.attn_output.weight", "offset": 48086958080, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.42.attn_q.weight", "offset": 48189718528, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.42.attn_v.weight", "offset": 48292478976, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.42.attn_norm.weight", "offset": 48307159040, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 48307187712, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.42.ffn_up.weight", "offset": 48600788992, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.42.ffn_norm.weight", "offset": 48894390272, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.43.attn_norm.weight", "offset": 48894418944, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 48894447616, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.43.ffn_gate.weight", "offset": 49188048896, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.43.ffn_up.weight", "offset": 49481650176, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.43.ffn_norm.weight", "offset": 49775251456, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 49775280128, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.43.attn_output.weight", "offset": 49789960192, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.43.attn_q.weight", "offset": 49892720640, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.43.attn_v.weight", "offset": 49995481088, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.44.attn_norm.weight", "offset": 50010161152, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 50010189824, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.44.ffn_gate.weight", "offset": 50303791104, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.44.ffn_up.weight", "offset": 50597392384, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.44.ffn_norm.weight", "offset": 50890993664, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 50891022336, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.44.attn_output.weight", "offset": 50905702400, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.44.attn_q.weight", "offset": 51008462848, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.44.attn_v.weight", "offset": 51111223296, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.45.attn_norm.weight", "offset": 51125903360, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 51125932032, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.45.ffn_gate.weight", "offset": 51419533312, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.45.ffn_up.weight", "offset": 51713134592, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.45.ffn_norm.weight", "offset": 52006735872, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 52006764544, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.45.attn_output.weight", "offset": 52021444608, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.45.attn_q.weight", "offset": 52124205056, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.45.attn_v.weight", "offset": 52226965504, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.46.ffn_gate.weight", "offset": 52241645568, "shape": [ 7168, 20480 ], "size": 14680064, "type": 1 }, { "name": "blk.46.ffn_up.weight", "offset": 52535246848, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.46.attn_k.weight", "offset": 52828848128, "shape": [ 7168, 1024 ], "size": 293601280, "type": 1 }, { "name": "blk.46.attn_output.weight", "offset": 52843528192, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.46.attn_q.weight", "offset": 52946288640, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.46.attn_v.weight", "offset": 53049049088, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.46.attn_norm.weight", "offset": 53063729152, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 53063757824, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.46.ffn_norm.weight", "offset": 53357359104, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.47.attn_norm.weight", "offset": 53357387776, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 53357416448, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.47.ffn_gate.weight", "offset": 53651017728, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.47.ffn_up.weight", "offset": 53944619008, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.47.ffn_norm.weight", "offset": 54238220288, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 54238248960, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.47.attn_output.weight", "offset": 54252929024, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.47.attn_q.weight", "offset": 54355689472, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.47.attn_v.weight", "offset": 54458449920, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.48.attn_norm.weight", "offset": 54473129984, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 54473158656, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.48.ffn_gate.weight", "offset": 54766759936, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.48.ffn_up.weight", "offset": 55060361216, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.48.ffn_norm.weight", "offset": 55353962496, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 55353991168, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.48.attn_output.weight", "offset": 55368671232, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.48.attn_q.weight", "offset": 55471431680, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.48.attn_v.weight", "offset": 55574192128, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.49.attn_norm.weight", "offset": 55588872192, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 55588900864, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.49.ffn_gate.weight", "offset": 55882502144, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.49.ffn_up.weight", "offset": 56176103424, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.49.ffn_norm.weight", "offset": 56469704704, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 56469733376, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.49.attn_output.weight", "offset": 56484413440, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.49.attn_q.weight", "offset": 56587173888, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.49.attn_v.weight", "offset": 56689934336, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.50.attn_norm.weight", "offset": 56704614400, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 56704643072, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.50.ffn_gate.weight", "offset": 56998244352, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.50.ffn_up.weight", "offset": 57291845632, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.50.ffn_norm.weight", "offset": 57585446912, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 57585475584, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.50.attn_output.weight", "offset": 57600155648, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.50.attn_q.weight", "offset": 57702916096, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.50.attn_v.weight", "offset": 57805676544, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.51.attn_k.weight", "offset": 57820356608, "shape": [ 7168, 1024 ], "size": 14680064, "type": 1 }, { "name": "blk.51.attn_output.weight", "offset": 57835036672, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.51.attn_q.weight", "offset": 57937797120, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.51.attn_v.weight", "offset": 58040557568, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.51.attn_norm.weight", "offset": 58055237632, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 58055266304, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.51.ffn_gate.weight", "offset": 58348867584, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.51.ffn_up.weight", "offset": 58642468864, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.51.ffn_norm.weight", "offset": 58936070144, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.52.attn_norm.weight", "offset": 58936098816, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 58936127488, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.52.ffn_gate.weight", "offset": 59229728768, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.52.ffn_up.weight", "offset": 59523330048, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.52.ffn_norm.weight", "offset": 59816931328, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 59816960000, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.52.attn_output.weight", "offset": 59831640064, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.52.attn_q.weight", "offset": 59934400512, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.52.attn_v.weight", "offset": 60037160960, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.53.attn_norm.weight", "offset": 60051841024, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 60051869696, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.53.ffn_gate.weight", "offset": 60345470976, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.53.ffn_up.weight", "offset": 60639072256, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.53.ffn_norm.weight", "offset": 60932673536, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 60932702208, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.53.attn_output.weight", "offset": 60947382272, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.53.attn_q.weight", "offset": 61050142720, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.53.attn_v.weight", "offset": 61152903168, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.54.attn_norm.weight", "offset": 61167583232, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 61167611904, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.54.ffn_gate.weight", "offset": 61461213184, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.54.ffn_up.weight", "offset": 61754814464, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.54.ffn_norm.weight", "offset": 62048415744, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 62048444416, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.54.attn_output.weight", "offset": 62063124480, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.54.attn_q.weight", "offset": 62165884928, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.54.attn_v.weight", "offset": 62268645376, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.55.ffn_gate.weight", "offset": 62283325440, "shape": [ 7168, 20480 ], "size": 14680064, "type": 1 }, { "name": "blk.55.attn_k.weight", "offset": 62576926720, "shape": [ 7168, 1024 ], "size": 293601280, "type": 1 }, { "name": "blk.55.attn_output.weight", "offset": 62591606784, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.55.attn_q.weight", "offset": 62694367232, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.55.attn_v.weight", "offset": 62797127680, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.55.attn_norm.weight", "offset": 62811807744, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 62811836416, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.55.ffn_up.weight", "offset": 63105437696, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.55.ffn_norm.weight", "offset": 63399038976, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.56.attn_norm.weight", "offset": 63399067648, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 63399096320, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.56.ffn_gate.weight", "offset": 63692697600, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.56.ffn_up.weight", "offset": 63986298880, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.56.ffn_norm.weight", "offset": 64279900160, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 64279928832, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.56.attn_output.weight", "offset": 64294608896, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.56.attn_q.weight", "offset": 64397369344, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.56.attn_v.weight", "offset": 64500129792, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.57.attn_norm.weight", "offset": 64514809856, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 64514838528, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.57.ffn_gate.weight", "offset": 64808439808, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.57.ffn_up.weight", "offset": 65102041088, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.57.ffn_norm.weight", "offset": 65395642368, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 65395671040, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.57.attn_output.weight", "offset": 65410351104, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.57.attn_q.weight", "offset": 65513111552, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.57.attn_v.weight", "offset": 65615872000, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.58.attn_norm.weight", "offset": 65630552064, "shape": [ 7168 ], "size": 14680064, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 65630580736, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.58.ffn_gate.weight", "offset": 65924182016, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.58.ffn_up.weight", "offset": 66217783296, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.58.ffn_norm.weight", "offset": 66511384576, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 66511413248, "shape": [ 7168, 1024 ], "size": 28672, "type": 1 }, { "name": "blk.58.attn_output.weight", "offset": 66526093312, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.58.attn_q.weight", "offset": 66628853760, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.58.attn_v.weight", "offset": 66731614208, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "blk.59.ffn_gate.weight", "offset": 66746294272, "shape": [ 7168, 20480 ], "size": 14680064, "type": 1 }, { "name": "blk.59.ffn_up.weight", "offset": 67039895552, "shape": [ 7168, 20480 ], "size": 293601280, "type": 1 }, { "name": "blk.59.attn_k.weight", "offset": 67333496832, "shape": [ 7168, 1024 ], "size": 293601280, "type": 1 }, { "name": "blk.59.attn_output.weight", "offset": 67348176896, "shape": [ 7168, 7168 ], "size": 14680064, "type": 1 }, { "name": "blk.59.attn_q.weight", "offset": 67450937344, "shape": [ 7168, 7168 ], "size": 102760448, "type": 1 }, { "name": "blk.59.attn_v.weight", "offset": 67553697792, "shape": [ 7168, 1024 ], "size": 102760448, "type": 1 }, { "name": "output.weight", "offset": 67568377856, "shape": [ 7168, 64000 ], "size": 14680064, "type": 1 }, { "name": "blk.59.attn_norm.weight", "offset": 68485881856, "shape": [ 7168 ], "size": 917504000, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 68485910528, "shape": [ 20480, 7168 ], "size": 28672, "type": 1 }, { "name": "blk.59.ffn_norm.weight", "offset": 68779511808, "shape": [ 7168 ], "size": 293601280, "type": 0 }, { "name": "output_norm.weight", "offset": 68779540480, "shape": [ 7168 ], "size": 28672, "type": 0 } ], "version": 3 }