yi

{ "metadata": { "general.architecture": "llama", "general.file_type": 18, "general.name": "Yi-1.5-34B-Chat", "general.quantization_version": 2, "llama.attention.head_count": 56, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 60, "llama.context_length": 4096, "llama.embedding_length": 7168, "llama.feed_forward_length": 20480, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 5000000, "llama.vocab_size": 64000, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (64000 values)", "tokenizer.ggml.token_type": "... (64000 values)", "tokenizer.ggml.tokens": "... (64000 values)" }, "num_params": 34388917248, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 376320000, "shape": [ 7168 ], "size": 376320000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 376348672, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 496771072, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 617193472, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.0.ffn_norm.weight", "offset": 737615872, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 737644544, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.0.attn_output.weight", "offset": 743665664, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.0.attn_q.weight", "offset": 785813504, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.0.attn_v.weight", "offset": 827961344, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 833982464, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 834011136, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 954433536, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 1074855936, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.1.ffn_norm.weight", "offset": 1195278336, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1195307008, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.1.attn_output.weight", "offset": 1201328128, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.1.attn_q.weight", "offset": 1243475968, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.1.attn_v.weight", "offset": 1285623808, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 1291644928, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1291673600, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 1412096000, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 1532518400, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.2.ffn_norm.weight", "offset": 1652940800, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1652969472, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.2.attn_output.weight", "offset": 1658990592, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.2.attn_q.weight", "offset": 1701138432, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.2.attn_v.weight", "offset": 1743286272, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 1749307392, "shape": [ 7168, 20480 ], "size": 6021120, "type": 14 }, { "name": "blk.3.attn_k.weight", "offset": 1869729792, "shape": [ 7168, 1024 ], "size": 120422400, "type": 14 }, { "name": "blk.3.attn_output.weight", "offset": 1875750912, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.3.attn_q.weight", "offset": 1917898752, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.3.attn_v.weight", "offset": 1960046592, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 1966067712, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1966096384, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.3.ffn_up.weight", "offset": 2086518784, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.3.ffn_norm.weight", "offset": 2206941184, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 2206969856, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 2206998528, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 2327420928, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 2447843328, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.4.ffn_norm.weight", "offset": 2568265728, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 2568294400, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.4.attn_output.weight", "offset": 2574315520, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.4.attn_q.weight", "offset": 2616463360, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.4.attn_v.weight", "offset": 2658611200, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 2664632320, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 2664660992, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.5.ffn_gate.weight", "offset": 2785083392, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.5.ffn_up.weight", "offset": 2905505792, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.5.ffn_norm.weight", "offset": 3025928192, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 3025956864, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.5.attn_output.weight", "offset": 3031977984, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.5.attn_q.weight", "offset": 3074125824, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.5.attn_v.weight", "offset": 3116273664, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 3122294784, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 3122323456, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.6.ffn_gate.weight", "offset": 3242745856, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.6.ffn_up.weight", "offset": 3363168256, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.6.ffn_norm.weight", "offset": 3483590656, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 3483619328, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.6.attn_output.weight", "offset": 3489640448, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.6.attn_q.weight", "offset": 3531788288, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.6.attn_v.weight", "offset": 3573936128, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 3579957248, "shape": [ 7168, 20480 ], "size": 6021120, "type": 14 }, { "name": "blk.7.ffn_up.weight", "offset": 3700379648, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.7.attn_k.weight", "offset": 3820802048, "shape": [ 7168, 1024 ], "size": 120422400, "type": 14 }, { "name": "blk.7.attn_output.weight", "offset": 3826823168, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.7.attn_q.weight", "offset": 3868971008, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.7.attn_v.weight", "offset": 3911118848, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 3917139968, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 3917168640, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 4037591040, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.10.ffn_up.weight", "offset": 4158013440, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.10.ffn_norm.weight", "offset": 4278435840, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 4278464512, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.10.attn_output.weight", "offset": 4284485632, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.10.attn_q.weight", "offset": 4326633472, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.10.attn_v.weight", "offset": 4368781312, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 4374802432, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 4374831104, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.11.ffn_gate.weight", "offset": 4495253504, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.11.ffn_up.weight", "offset": 4615675904, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.11.ffn_norm.weight", "offset": 4736098304, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 4736126976, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.11.attn_output.weight", "offset": 4742148096, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.11.attn_q.weight", "offset": 4784295936, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "\u0000\u001c\u0000\u0000\u0000\u0000\u0000\u0000\u000e\u0000\u0000\u0000\u0000\ufffd\ufffd\ufffd\u0002\u0000\u0000\u0000", "offset": 4826443776, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.12.attn_k.weight", "offset": 4832464896, "shape": [ 7168, 1024 ], "size": 6021120, "type": 14 }, { "name": "blk.12.attn_output.weight", "offset": 4838486016, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.12.attn_q.weight", "offset": 4880633856, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.12.attn_v.weight", "offset": 4922781696, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 4928802816, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 4928831488, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.7.ffn_norm.weight", "offset": 5049253888, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 5049282560, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 5049311232, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.8.ffn_gate.weight", "offset": 5169733632, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.8.ffn_up.weight", "offset": 5290156032, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.8.ffn_norm.weight", "offset": 5410578432, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 5410607104, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.8.attn_output.weight", "offset": 5416628224, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.8.attn_q.weight", "offset": 5458776064, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.8.attn_v.weight", "offset": 5500923904, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 5506945024, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 5506973696, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 5627396096, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.9.ffn_up.weight", "offset": 5747818496, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.9.ffn_norm.weight", "offset": 5868240896, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 5868269568, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.9.attn_output.weight", "offset": 5874290688, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.9.attn_q.weight", "offset": 5916438528, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.9.attn_v.weight", "offset": 5958586368, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 5964607488, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 5964636160, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.12.ffn_gate.weight", "offset": 6085058560, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.12.ffn_up.weight", "offset": 6205480960, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.12.ffn_norm.weight", "offset": 6325903360, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 6325932032, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 6325960704, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.13.ffn_gate.weight", "offset": 6446383104, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.13.ffn_up.weight", "offset": 6566805504, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.13.ffn_norm.weight", "offset": 6687227904, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 6687256576, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.13.attn_output.weight", "offset": 6693277696, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.13.attn_q.weight", "offset": 6735425536, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.13.attn_v.weight", "offset": 6777573376, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 6783594496, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 6783623168, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.14.ffn_gate.weight", "offset": 6904045568, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.14.ffn_up.weight", "offset": 7024467968, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.14.ffn_norm.weight", "offset": 7144890368, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 7144919040, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.14.attn_output.weight", "offset": 7150940160, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.14.attn_q.weight", "offset": 7193088000, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.14.attn_v.weight", "offset": 7235235840, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.15.attn_norm.weight", "offset": 7241256960, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 7241285632, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.15.ffn_gate.weight", "offset": 7361708032, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.15.ffn_up.weight", "offset": 7482130432, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.15.ffn_norm.weight", "offset": 7602552832, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 7602581504, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.15.attn_output.weight", "offset": 7608602624, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.15.attn_q.weight", "offset": 7650750464, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.15.attn_v.weight", "offset": 7692898304, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 7698919424, "shape": [ 7168, 20480 ], "size": 6021120, "type": 14 }, { "name": "blk.16.attn_k.weight", "offset": 7819341824, "shape": [ 7168, 1024 ], "size": 120422400, "type": 14 }, { "name": "blk.16.attn_output.weight", "offset": 7825362944, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.16.attn_q.weight", "offset": 7867510784, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.16.attn_v.weight", "offset": 7909658624, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 7915679744, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 7915708416, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.16.ffn_up.weight", "offset": 8036130816, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.16.ffn_norm.weight", "offset": 8156553216, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 8156581888, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 8156610560, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.17.ffn_gate.weight", "offset": 8277032960, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.17.ffn_up.weight", "offset": 8397455360, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.17.ffn_norm.weight", "offset": 8517877760, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 8517906432, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.17.attn_output.weight", "offset": 8523927552, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.17.attn_q.weight", "offset": 8566075392, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.17.attn_v.weight", "offset": 8608223232, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 8614244352, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 8614273024, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.18.ffn_gate.weight", "offset": 8734695424, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.18.ffn_up.weight", "offset": 8855117824, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.18.ffn_norm.weight", "offset": 8975540224, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 8975568896, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.18.attn_output.weight", "offset": 8981590016, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.18.attn_q.weight", "offset": 9023737856, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.18.attn_v.weight", "offset": 9065885696, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 9071906816, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 9071935488, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.19.ffn_gate.weight", "offset": 9192357888, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 9312780288, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.19.ffn_norm.weight", "offset": 9433202688, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 9433231360, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.19.attn_output.weight", "offset": 9439252480, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.19.attn_q.weight", "offset": 9481400320, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.19.attn_v.weight", "offset": 9523548160, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.20.ffn_gate.weight", "offset": 9529569280, "shape": [ 7168, 20480 ], "size": 6021120, "type": 14 }, { "name": "blk.20.ffn_up.weight", "offset": 9649991680, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.20.attn_k.weight", "offset": 9770414080, "shape": [ 7168, 1024 ], "size": 120422400, "type": 14 }, { "name": "blk.20.attn_output.weight", "offset": 9776435200, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.20.attn_q.weight", "offset": 9818583040, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.20.attn_v.weight", "offset": 9860730880, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.20.attn_norm.weight", "offset": 9866752000, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 9866780672, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.20.ffn_norm.weight", "offset": 9987203072, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.21.attn_norm.weight", "offset": 9987231744, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 9987260416, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.21.ffn_gate.weight", "offset": 10107682816, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.21.ffn_up.weight", "offset": 10228105216, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.21.ffn_norm.weight", "offset": 10348527616, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 10348556288, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.21.attn_output.weight", "offset": 10354577408, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.21.attn_q.weight", "offset": 10396725248, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.21.attn_v.weight", "offset": 10438873088, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 10444894208, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 10444922880, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.22.ffn_gate.weight", "offset": 10565345280, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.22.ffn_up.weight", "offset": 10685767680, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.22.ffn_norm.weight", "offset": 10806190080, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 10806218752, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.22.attn_output.weight", "offset": 10812239872, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.22.attn_q.weight", "offset": 10854387712, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.22.attn_v.weight", "offset": 10896535552, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.23.attn_norm.weight", "offset": 10902556672, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 10902585344, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.23.ffn_gate.weight", "offset": 11023007744, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.23.ffn_up.weight", "offset": 11143430144, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.23.ffn_norm.weight", "offset": 11263852544, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 11263881216, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.23.attn_output.weight", "offset": 11269902336, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.23.attn_q.weight", "offset": 11312050176, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.23.attn_v.weight", "offset": 11354198016, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 11360219136, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 11360247808, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.24.ffn_gate.weight", "offset": 11480670208, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.24.ffn_up.weight", "offset": 11601092608, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.24.ffn_norm.weight", "offset": 11721515008, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 11721543680, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.24.attn_output.weight", "offset": 11727564800, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.24.attn_q.weight", "offset": 11769712640, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.24.attn_v.weight", "offset": 11811860480, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.25.attn_k.weight", "offset": 11817881600, "shape": [ 7168, 1024 ], "size": 6021120, "type": 14 }, { "name": "blk.25.attn_output.weight", "offset": 11823902720, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.25.attn_q.weight", "offset": 11866050560, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.25.attn_v.weight", "offset": 11908198400, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 11914219520, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 11914248192, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 12034670592, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.25.ffn_up.weight", "offset": 12155092992, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.25.ffn_norm.weight", "offset": 12275515392, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.26.attn_norm.weight", "offset": 12275544064, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 12275572736, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.26.ffn_gate.weight", "offset": 12395995136, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.26.ffn_up.weight", "offset": 12516417536, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.26.ffn_norm.weight", "offset": 12636839936, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 12636868608, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.26.attn_output.weight", "offset": 12642889728, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.26.attn_q.weight", "offset": 12685037568, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.26.attn_v.weight", "offset": 12727185408, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 12733206528, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 12733235200, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 12853657600, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.27.ffn_up.weight", "offset": 12974080000, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.27.ffn_norm.weight", "offset": 13094502400, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 13094531072, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.27.attn_output.weight", "offset": 13100552192, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.27.attn_q.weight", "offset": 13142700032, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.27.attn_v.weight", "offset": 13184847872, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.28.attn_norm.weight", "offset": 13190868992, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 13190897664, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.28.ffn_gate.weight", "offset": 13311320064, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.28.ffn_up.weight", "offset": 13431742464, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.28.ffn_norm.weight", "offset": 13552164864, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 13552193536, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.28.attn_output.weight", "offset": 13558214656, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.28.attn_q.weight", "offset": 13600362496, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.28.attn_v.weight", "offset": 13642510336, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.29.ffn_gate.weight", "offset": 13648531456, "shape": [ 7168, 20480 ], "size": 6021120, "type": 14 }, { "name": "blk.29.attn_k.weight", "offset": 13768953856, "shape": [ 7168, 1024 ], "size": 120422400, "type": 14 }, { "name": "blk.29.attn_output.weight", "offset": 13774974976, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.29.attn_q.weight", "offset": 13817122816, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.29.attn_v.weight", "offset": 13859270656, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.29.attn_norm.weight", "offset": 13865291776, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 13865320448, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.29.ffn_up.weight", "offset": 13985742848, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.29.ffn_norm.weight", "offset": 14106165248, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.30.attn_norm.weight", "offset": 14106193920, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 14106222592, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.30.ffn_gate.weight", "offset": 14226644992, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.30.ffn_up.weight", "offset": 14347067392, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.30.ffn_norm.weight", "offset": 14467489792, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 14467518464, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.30.attn_output.weight", "offset": 14473539584, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.30.attn_q.weight", "offset": 14515687424, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.30.attn_v.weight", "offset": 14557835264, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.31.attn_norm.weight", "offset": 14563856384, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 14563885056, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.31.ffn_gate.weight", "offset": 14684307456, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.31.ffn_up.weight", "offset": 14804729856, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.31.ffn_norm.weight", "offset": 14925152256, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 14925180928, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.31.attn_output.weight", "offset": 14931202048, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.31.attn_q.weight", "offset": 14973349888, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.31.attn_v.weight", "offset": 15015497728, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.32.attn_norm.weight", "offset": 15021518848, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 15021547520, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.32.ffn_gate.weight", "offset": 15141969920, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.32.ffn_up.weight", "offset": 15262392320, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.32.ffn_norm.weight", "offset": 15382814720, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 15382843392, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.32.attn_output.weight", "offset": 15388864512, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.32.attn_q.weight", "offset": 15431012352, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.32.attn_v.weight", "offset": 15473160192, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.33.ffn_gate.weight", "offset": 15479181312, "shape": [ 7168, 20480 ], "size": 6021120, "type": 14 }, { "name": "blk.33.ffn_up.weight", "offset": 15599603712, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.33.attn_k.weight", "offset": 15720026112, "shape": [ 7168, 1024 ], "size": 120422400, "type": 14 }, { "name": "blk.33.attn_output.weight", "offset": 15726047232, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.33.attn_q.weight", "offset": 15768195072, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.33.attn_v.weight", "offset": 15810342912, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.33.attn_norm.weight", "offset": 15816364032, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 15816392704, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.33.ffn_norm.weight", "offset": 15936815104, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.34.attn_norm.weight", "offset": 15936843776, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 15936872448, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.34.ffn_gate.weight", "offset": 16057294848, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.34.ffn_up.weight", "offset": 16177717248, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.34.ffn_norm.weight", "offset": 16298139648, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 16298168320, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.34.attn_output.weight", "offset": 16304189440, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.34.attn_q.weight", "offset": 16346337280, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.34.attn_v.weight", "offset": 16388485120, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 16394506240, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 16394534912, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.35.ffn_gate.weight", "offset": 16514957312, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.35.ffn_up.weight", "offset": 16635379712, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.35.ffn_norm.weight", "offset": 16755802112, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 16755830784, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.35.attn_output.weight", "offset": 16761851904, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.35.attn_q.weight", "offset": 16803999744, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.35.attn_v.weight", "offset": 16846147584, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.36.attn_norm.weight", "offset": 16852168704, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 16852197376, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.36.ffn_gate.weight", "offset": 16972619776, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.36.ffn_up.weight", "offset": 17093042176, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.36.ffn_norm.weight", "offset": 17213464576, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 17213493248, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.36.attn_output.weight", "offset": 17219514368, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.36.attn_q.weight", "offset": 17261662208, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.36.attn_v.weight", "offset": 17303810048, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.37.attn_norm.weight", "offset": 17309831168, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 17309859840, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.37.ffn_gate.weight", "offset": 17430282240, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.37.ffn_up.weight", "offset": 17550704640, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.37.ffn_norm.weight", "offset": 17671127040, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 17671155712, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.37.attn_output.weight", "offset": 17677176832, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.37.attn_q.weight", "offset": 17719324672, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.37.attn_v.weight", "offset": 17761472512, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.38.attn_k.weight", "offset": 17767493632, "shape": [ 7168, 1024 ], "size": 6021120, "type": 14 }, { "name": "blk.38.attn_output.weight", "offset": 17773514752, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.38.attn_q.weight", "offset": 17815662592, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.38.attn_v.weight", "offset": 17857810432, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 17863831552, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 17863860224, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.38.ffn_gate.weight", "offset": 17984282624, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.38.ffn_up.weight", "offset": 18104705024, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.38.ffn_norm.weight", "offset": 18225127424, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.39.attn_norm.weight", "offset": 18225156096, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 18225184768, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.39.ffn_gate.weight", "offset": 18345607168, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.39.ffn_up.weight", "offset": 18466029568, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.39.ffn_norm.weight", "offset": 18586451968, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 18586480640, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.39.attn_output.weight", "offset": 18592501760, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.39.attn_q.weight", "offset": 18634649600, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.39.attn_v.weight", "offset": 18676797440, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.40.attn_norm.weight", "offset": 18682818560, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 18682847232, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.40.ffn_gate.weight", "offset": 18803269632, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.40.ffn_up.weight", "offset": 18923692032, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.40.ffn_norm.weight", "offset": 19044114432, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 19044143104, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.40.attn_output.weight", "offset": 19050164224, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.40.attn_q.weight", "offset": 19092312064, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.40.attn_v.weight", "offset": 19134459904, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 19140481024, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 19140509696, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.41.ffn_gate.weight", "offset": 19260932096, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.41.ffn_up.weight", "offset": 19381354496, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.41.ffn_norm.weight", "offset": 19501776896, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 19501805568, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.41.attn_output.weight", "offset": 19507826688, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.41.attn_q.weight", "offset": 19549974528, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.41.attn_v.weight", "offset": 19592122368, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.42.ffn_gate.weight", "offset": 19598143488, "shape": [ 7168, 20480 ], "size": 6021120, "type": 14 }, { "name": "blk.42.attn_k.weight", "offset": 19718565888, "shape": [ 7168, 1024 ], "size": 120422400, "type": 14 }, { "name": "blk.42.attn_output.weight", "offset": 19724587008, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.42.attn_q.weight", "offset": 19766734848, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.42.attn_v.weight", "offset": 19808882688, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.42.attn_norm.weight", "offset": 19814903808, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 19814932480, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.42.ffn_up.weight", "offset": 19935354880, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.42.ffn_norm.weight", "offset": 20055777280, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.43.attn_norm.weight", "offset": 20055805952, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 20055834624, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.43.ffn_gate.weight", "offset": 20176257024, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.43.ffn_up.weight", "offset": 20296679424, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.43.ffn_norm.weight", "offset": 20417101824, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 20417130496, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.43.attn_output.weight", "offset": 20423151616, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.43.attn_q.weight", "offset": 20465299456, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.43.attn_v.weight", "offset": 20507447296, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.44.attn_norm.weight", "offset": 20513468416, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 20513497088, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.44.ffn_gate.weight", "offset": 20633919488, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.44.ffn_up.weight", "offset": 20754341888, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.44.ffn_norm.weight", "offset": 20874764288, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 20874792960, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.44.attn_output.weight", "offset": 20880814080, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.44.attn_q.weight", "offset": 20922961920, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.44.attn_v.weight", "offset": 20965109760, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.45.attn_norm.weight", "offset": 20971130880, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 20971159552, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.45.ffn_gate.weight", "offset": 21091581952, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.45.ffn_up.weight", "offset": 21212004352, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.45.ffn_norm.weight", "offset": 21332426752, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 21332455424, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.45.attn_output.weight", "offset": 21338476544, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.45.attn_q.weight", "offset": 21380624384, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.45.attn_v.weight", "offset": 21422772224, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.46.ffn_gate.weight", "offset": 21428793344, "shape": [ 7168, 20480 ], "size": 6021120, "type": 14 }, { "name": "blk.46.ffn_up.weight", "offset": 21549215744, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.46.attn_k.weight", "offset": 21669638144, "shape": [ 7168, 1024 ], "size": 120422400, "type": 14 }, { "name": "blk.46.attn_output.weight", "offset": 21675659264, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.46.attn_q.weight", "offset": 21717807104, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.46.attn_v.weight", "offset": 21759954944, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.46.attn_norm.weight", "offset": 21765976064, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 21766004736, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.46.ffn_norm.weight", "offset": 21886427136, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.47.attn_norm.weight", "offset": 21886455808, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 21886484480, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.47.ffn_gate.weight", "offset": 22006906880, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.47.ffn_up.weight", "offset": 22127329280, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.47.ffn_norm.weight", "offset": 22247751680, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 22247780352, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.47.attn_output.weight", "offset": 22253801472, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.47.attn_q.weight", "offset": 22295949312, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.47.attn_v.weight", "offset": 22338097152, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.48.attn_norm.weight", "offset": 22344118272, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 22344146944, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.48.ffn_gate.weight", "offset": 22464569344, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.48.ffn_up.weight", "offset": 22584991744, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.48.ffn_norm.weight", "offset": 22705414144, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 22705442816, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.48.attn_output.weight", "offset": 22711463936, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.48.attn_q.weight", "offset": 22753611776, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.48.attn_v.weight", "offset": 22795759616, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.49.attn_norm.weight", "offset": 22801780736, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 22801809408, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.49.ffn_gate.weight", "offset": 22922231808, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.49.ffn_up.weight", "offset": 23042654208, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.49.ffn_norm.weight", "offset": 23163076608, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 23163105280, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.49.attn_output.weight", "offset": 23169126400, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.49.attn_q.weight", "offset": 23211274240, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.49.attn_v.weight", "offset": 23253422080, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.50.attn_norm.weight", "offset": 23259443200, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 23259471872, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.50.ffn_gate.weight", "offset": 23379894272, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.50.ffn_up.weight", "offset": 23500316672, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.50.ffn_norm.weight", "offset": 23620739072, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 23620767744, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.50.attn_output.weight", "offset": 23626788864, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.50.attn_q.weight", "offset": 23668936704, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.50.attn_v.weight", "offset": 23711084544, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.51.attn_k.weight", "offset": 23717105664, "shape": [ 7168, 1024 ], "size": 6021120, "type": 14 }, { "name": "blk.51.attn_output.weight", "offset": 23723126784, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.51.attn_q.weight", "offset": 23765274624, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.51.attn_v.weight", "offset": 23807422464, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.51.attn_norm.weight", "offset": 23813443584, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 23813472256, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.51.ffn_gate.weight", "offset": 23933894656, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.51.ffn_up.weight", "offset": 24054317056, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.51.ffn_norm.weight", "offset": 24174739456, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.52.attn_norm.weight", "offset": 24174768128, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 24174796800, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.52.ffn_gate.weight", "offset": 24295219200, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.52.ffn_up.weight", "offset": 24415641600, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.52.ffn_norm.weight", "offset": 24536064000, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 24536092672, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.52.attn_output.weight", "offset": 24542113792, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.52.attn_q.weight", "offset": 24584261632, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.52.attn_v.weight", "offset": 24626409472, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.53.attn_norm.weight", "offset": 24632430592, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 24632459264, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.53.ffn_gate.weight", "offset": 24752881664, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.53.ffn_up.weight", "offset": 24873304064, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.53.ffn_norm.weight", "offset": 24993726464, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 24993755136, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.53.attn_output.weight", "offset": 24999776256, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.53.attn_q.weight", "offset": 25041924096, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.53.attn_v.weight", "offset": 25084071936, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.54.attn_norm.weight", "offset": 25090093056, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 25090121728, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.54.ffn_gate.weight", "offset": 25210544128, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.54.ffn_up.weight", "offset": 25330966528, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.54.ffn_norm.weight", "offset": 25451388928, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 25451417600, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.54.attn_output.weight", "offset": 25457438720, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.54.attn_q.weight", "offset": 25499586560, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.54.attn_v.weight", "offset": 25541734400, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.55.ffn_gate.weight", "offset": 25547755520, "shape": [ 7168, 20480 ], "size": 6021120, "type": 14 }, { "name": "blk.55.attn_k.weight", "offset": 25668177920, "shape": [ 7168, 1024 ], "size": 120422400, "type": 14 }, { "name": "blk.55.attn_output.weight", "offset": 25674199040, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.55.attn_q.weight", "offset": 25716346880, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.55.attn_v.weight", "offset": 25758494720, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.55.attn_norm.weight", "offset": 25764515840, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 25764544512, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.55.ffn_up.weight", "offset": 25884966912, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.55.ffn_norm.weight", "offset": 26005389312, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.56.attn_norm.weight", "offset": 26005417984, "shape": [ 7168 ], "size": 28672, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 26005446656, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.56.ffn_gate.weight", "offset": 26125869056, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.56.ffn_up.weight", "offset": 26246291456, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.56.ffn_norm.weight", "offset": 26366713856, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 26366742528, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.56.attn_output.weight", "offset": 26372763648, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.56.attn_q.weight", "offset": 26414911488, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.56.attn_v.weight", "offset": 26457059328, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.57.attn_norm.weight", "offset": 26463080448, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 26463109120, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.57.ffn_gate.weight", "offset": 26583531520, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.57.ffn_up.weight", "offset": 26703953920, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.57.ffn_norm.weight", "offset": 26824376320, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.57.attn_k.weight", "offset": 26824404992, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.57.attn_output.weight", "offset": 26830426112, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.57.attn_q.weight", "offset": 26872573952, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.57.attn_v.weight", "offset": 26914721792, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.58.attn_norm.weight", "offset": 26920742912, "shape": [ 7168 ], "size": 6021120, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 26920771584, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.58.ffn_gate.weight", "offset": 27041193984, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.58.ffn_up.weight", "offset": 27161616384, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.58.ffn_norm.weight", "offset": 27282038784, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 27282067456, "shape": [ 7168, 1024 ], "size": 28672, "type": 14 }, { "name": "blk.58.attn_output.weight", "offset": 27288088576, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.58.attn_q.weight", "offset": 27330236416, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.58.attn_v.weight", "offset": 27372384256, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "blk.59.ffn_gate.weight", "offset": 27378405376, "shape": [ 7168, 20480 ], "size": 6021120, "type": 14 }, { "name": "blk.59.ffn_up.weight", "offset": 27498827776, "shape": [ 7168, 20480 ], "size": 120422400, "type": 14 }, { "name": "blk.59.attn_k.weight", "offset": 27619250176, "shape": [ 7168, 1024 ], "size": 120422400, "type": 14 }, { "name": "blk.59.attn_output.weight", "offset": 27625271296, "shape": [ 7168, 7168 ], "size": 6021120, "type": 14 }, { "name": "blk.59.attn_q.weight", "offset": 27667419136, "shape": [ 7168, 7168 ], "size": 42147840, "type": 14 }, { "name": "blk.59.attn_v.weight", "offset": 27709566976, "shape": [ 7168, 1024 ], "size": 42147840, "type": 14 }, { "name": "output.weight", "offset": 27715588096, "shape": [ 7168, 64000 ], "size": 6021120, "type": 14 }, { "name": "blk.59.attn_norm.weight", "offset": 28091908096, "shape": [ 7168 ], "size": 376320000, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 28091936768, "shape": [ 20480, 7168 ], "size": 28672, "type": 14 }, { "name": "blk.59.ffn_norm.weight", "offset": 28212359168, "shape": [ 7168 ], "size": 120422400, "type": 0 }, { "name": "output_norm.weight", "offset": 28212387840, "shape": [ 7168 ], "size": 28672, "type": 0 } ], "version": 3 }