dolphin-llama3

{ "metadata": { "general.architecture": "llama", "general.file_type": 7, "general.name": "dolphin-2.9-llama3-70b", "general.quantization_version": 2, "llama.attention.head_count": 64, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 80, "llama.context_length": 8192, "llama.embedding_length": 8192, "llama.feed_forward_length": 28672, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 500000, "llama.vocab_size": 128258, "tokenizer.ggml.bos_token_id": 128000, "tokenizer.ggml.eos_token_id": 128256, "tokenizer.ggml.merges": "... (280147 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 128001, "tokenizer.ggml.token_type": "... (128258 values)", "tokenizer.ggml.tokens": "... (128258 values)" }, "num_params": 70553739264, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 1116357632, "shape": [ 8192 ], "size": 1116357632, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 1116390400, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.0.ffn_gate.weight", "offset": 1365951488, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 1615512576, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 1865073664, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 1865106432, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 1874019328, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.0.attn_q.weight", "offset": 1945322496, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.0.attn_v.weight", "offset": 2016625664, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.1.ffn_gate.weight", "offset": 2025538560, "shape": [ 8192, 28672 ], "size": 8912896, "type": 8 }, { "name": "\u0000\u0000\u0000 \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd:\ufffd\u0003", "offset": 2275099648, "shape": [ 8192, 1024 ], "size": 249561088, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 2284012544, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.1.attn_q.weight", "offset": 2355315712, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.1.attn_v.weight", "offset": 2426618880, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 2435531776, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 2435564544, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.1.ffn_up.weight", "offset": 2685125632, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.1.ffn_norm.weight", "offset": 2934686720, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.2.attn_norm.weight", "offset": 2934719488, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 2934752256, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.2.ffn_gate.weight", "offset": 3184313344, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.2.ffn_up.weight", "offset": 3433874432, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.2.ffn_norm.weight", "offset": 3683435520, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 3683468288, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 3692381184, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.2.attn_q.weight", "offset": 3763684352, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.2.attn_v.weight", "offset": 3834987520, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.3.attn_norm.weight", "offset": 3843900416, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 3843933184, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.3.ffn_gate.weight", "offset": 4093494272, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.3.ffn_up.weight", "offset": 4343055360, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.3.ffn_norm.weight", "offset": 4592616448, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 4592649216, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 4601562112, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.3.attn_q.weight", "offset": 4672865280, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.3.attn_v.weight", "offset": 4744168448, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.4.attn_k.weight", "offset": 4753081344, "shape": [ 8192, 1024 ], "size": 8912896, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 4761994240, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.4.attn_q.weight", "offset": 4833297408, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.4.attn_v.weight", "offset": 4904600576, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 4913513472, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 4913546240, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.4.ffn_gate.weight", "offset": 5163107328, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.4.ffn_up.weight", "offset": 5412668416, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.4.ffn_norm.weight", "offset": 5662229504, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 5662262272, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 5662295040, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.5.ffn_gate.weight", "offset": 5911856128, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.5.ffn_up.weight", "offset": 6161417216, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.5.ffn_norm.weight", "offset": 6410978304, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 6411011072, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 6419923968, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.5.attn_q.weight", "offset": 6491227136, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.5.attn_v.weight", "offset": 6562530304, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.6.attn_norm.weight", "offset": 6571443200, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 6571475968, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.6.ffn_gate.weight", "offset": 6821037056, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.6.ffn_up.weight", "offset": 7070598144, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.6.ffn_norm.weight", "offset": 7320159232, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 7320192000, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 7329104896, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.6.attn_q.weight", "offset": 7400408064, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.6.attn_v.weight", "offset": 7471711232, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.7.attn_k.weight", "offset": 7480624128, "shape": [ 8192, 1024 ], "size": 8912896, "type": 8 }, { "name": "blk.7.attn_q.weight", "offset": 7489537024, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.7.attn_v.weight", "offset": 7560840192, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.7.attn_norm.weight", "offset": 7569753088, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 7569785856, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.7.ffn_gate.weight", "offset": 7819346944, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 8068908032, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.7.ffn_norm.weight", "offset": 8318469120, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 8318501888, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.8.attn_norm.weight", "offset": 8389805056, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 8389837824, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.8.ffn_gate.weight", "offset": 8639398912, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.8.ffn_up.weight", "offset": 8888960000, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.8.ffn_norm.weight", "offset": 9138521088, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 9138553856, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 9147466752, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.8.attn_q.weight", "offset": 9218769920, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.8.attn_v.weight", "offset": 9290073088, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 9298985984, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 9299018752, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.9.ffn_gate.weight", "offset": 9548579840, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.9.ffn_up.weight", "offset": 9798140928, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.9.ffn_norm.weight", "offset": 10047702016, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 10047734784, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 10056647680, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.9.attn_q.weight", "offset": 10127950848, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.9.attn_v.weight", "offset": 10199254016, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 10208166912, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 10208199680, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.10.ffn_gate.weight", "offset": 10457760768, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.10.ffn_up.weight", "offset": 10707321856, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.10.ffn_norm.weight", "offset": 10956882944, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 10956915712, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 10965828608, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.10.attn_q.weight", "offset": 11037131776, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.10.attn_v.weight", "offset": 11108434944, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 11117347840, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 11117380608, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.11.ffn_gate.weight", "offset": 11366941696, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.11.ffn_up.weight", "offset": 11616502784, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.11.ffn_norm.weight", "offset": 11866063872, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 11866096640, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 11875009536, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.11.attn_q.weight", "offset": 11946312704, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.11.attn_v.weight", "offset": 12017615872, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.12.ffn_gate.weight", "offset": 12026528768, "shape": [ 8192, 28672 ], "size": 8912896, "type": 8 }, { "name": "blk.12.ffn_up.weight", "offset": 12276089856, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.12.attn_k.weight", "offset": 12525650944, "shape": [ 8192, 1024 ], "size": 249561088, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 12534563840, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.12.attn_q.weight", "offset": 12605867008, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.12.attn_v.weight", "offset": 12677170176, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.12.attn_norm.weight", "offset": 12686083072, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 12686115840, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.12.ffn_norm.weight", "offset": 12935676928, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 12935709696, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 12935742464, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.13.ffn_gate.weight", "offset": 13185303552, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.13.ffn_up.weight", "offset": 13434864640, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.13.ffn_norm.weight", "offset": 13684425728, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 13684458496, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 13693371392, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.13.attn_q.weight", "offset": 13764674560, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.13.attn_v.weight", "offset": 13835977728, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 13844890624, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 13844923392, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.14.ffn_gate.weight", "offset": 14094484480, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.14.ffn_up.weight", "offset": 14344045568, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.14.ffn_norm.weight", "offset": 14593606656, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 14593639424, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 14602552320, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.14.attn_q.weight", "offset": 14673855488, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.14.attn_v.weight", "offset": 14745158656, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.15.ffn_gate.weight", "offset": 14754071552, "shape": [ 8192, 28672 ], "size": 8912896, "type": 8 }, { "name": "blk.15.attn_k.weight", "offset": 15003632640, "shape": [ 8192, 1024 ], "size": 249561088, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 15012545536, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.15.attn_q.weight", "offset": 15083848704, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.15.attn_v.weight", "offset": 15155151872, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.15.attn_norm.weight", "offset": 15164064768, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 15164097536, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.15.ffn_up.weight", "offset": 15413658624, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.15.ffn_norm.weight", "offset": 15663219712, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 15663252480, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 15663285248, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.16.ffn_gate.weight", "offset": 15912846336, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.16.ffn_up.weight", "offset": 16162407424, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.16.ffn_norm.weight", "offset": 16411968512, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 16412001280, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 16420914176, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.16.attn_q.weight", "offset": 16492217344, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.16.attn_v.weight", "offset": 16563520512, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 16572433408, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 16572466176, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.17.ffn_gate.weight", "offset": 16822027264, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.17.ffn_up.weight", "offset": 17071588352, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.17.ffn_norm.weight", "offset": 17321149440, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 17321182208, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 17330095104, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.17.attn_q.weight", "offset": 17401398272, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.17.attn_v.weight", "offset": 17472701440, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.18.attn_k.weight", "offset": 17481614336, "shape": [ 8192, 1024 ], "size": 8912896, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 17490527232, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.18.attn_q.weight", "offset": 17561830400, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.18.attn_v.weight", "offset": 17633133568, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.18.attn_norm.weight", "offset": 17642046464, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 17642079232, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.18.ffn_gate.weight", "offset": 17891640320, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.18.ffn_up.weight", "offset": 18141201408, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 18390762496, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 18390795264, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 18390828032, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.19.ffn_gate.weight", "offset": 18640389120, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.19.ffn_up.weight", "offset": 18889950208, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.19.ffn_norm.weight", "offset": 19139511296, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 19139544064, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 19148456960, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.19.attn_q.weight", "offset": 19219760128, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.19.attn_v.weight", "offset": 19291063296, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.20.attn_norm.weight", "offset": 19299976192, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 19300008960, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.20.ffn_gate.weight", "offset": 19549570048, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.20.ffn_up.weight", "offset": 19799131136, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.20.ffn_norm.weight", "offset": 20048692224, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 20048724992, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 20057637888, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.20.attn_q.weight", "offset": 20128941056, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.20.attn_v.weight", "offset": 20200244224, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.21.attn_k.weight", "offset": 20209157120, "shape": [ 8192, 1024 ], "size": 8912896, "type": 8 }, { "name": "blk.21.attn_q.weight", "offset": 20218070016, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.21.attn_v.weight", "offset": 20289373184, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.21.attn_norm.weight", "offset": 20298286080, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 20298318848, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.21.ffn_gate.weight", "offset": 20547879936, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.21.ffn_up.weight", "offset": 20797441024, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.21.ffn_norm.weight", "offset": 21047002112, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.21.attn_output.weight", "offset": 21047034880, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 21118338048, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 21118370816, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.22.ffn_gate.weight", "offset": 21367931904, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.22.ffn_up.weight", "offset": 21617492992, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.22.ffn_norm.weight", "offset": 21867054080, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 21867086848, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 21875999744, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.22.attn_q.weight", "offset": 21947302912, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.22.attn_v.weight", "offset": 22018606080, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.23.attn_norm.weight", "offset": 22027518976, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 22027551744, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.23.ffn_gate.weight", "offset": 22277112832, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.23.ffn_up.weight", "offset": 22526673920, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.23.ffn_norm.weight", "offset": 22776235008, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 22776267776, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 22785180672, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.23.attn_q.weight", "offset": 22856483840, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.23.attn_v.weight", "offset": 22927787008, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.24.attn_norm.weight", "offset": 22936699904, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 22936732672, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.24.ffn_gate.weight", "offset": 23186293760, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.24.ffn_up.weight", "offset": 23435854848, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.24.ffn_norm.weight", "offset": 23685415936, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 23685448704, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 23694361600, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.24.attn_q.weight", "offset": 23765664768, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.24.attn_v.weight", "offset": 23836967936, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.25.attn_norm.weight", "offset": 23845880832, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 23845913600, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.25.ffn_gate.weight", "offset": 24095474688, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.25.ffn_up.weight", "offset": 24345035776, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.25.ffn_norm.weight", "offset": 24594596864, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 24594629632, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 24603542528, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.25.attn_q.weight", "offset": 24674845696, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.25.attn_v.weight", "offset": 24746148864, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.26.ffn_gate.weight", "offset": 24755061760, "shape": [ 8192, 28672 ], "size": 8912896, "type": 8 }, { "name": "blk.26.ffn_up.weight", "offset": 25004622848, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.26.attn_k.weight", "offset": 25254183936, "shape": [ 8192, 1024 ], "size": 249561088, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 25263096832, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.26.attn_q.weight", "offset": 25334400000, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.26.attn_v.weight", "offset": 25405703168, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.26.attn_norm.weight", "offset": 25414616064, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 25414648832, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.26.ffn_norm.weight", "offset": 25664209920, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.27.attn_norm.weight", "offset": 25664242688, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 25664275456, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.27.ffn_gate.weight", "offset": 25913836544, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.27.ffn_up.weight", "offset": 26163397632, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.27.ffn_norm.weight", "offset": 26412958720, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 26412991488, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 26421904384, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.27.attn_q.weight", "offset": 26493207552, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.27.attn_v.weight", "offset": 26564510720, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.28.attn_norm.weight", "offset": 26573423616, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 26573456384, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.28.ffn_gate.weight", "offset": 26823017472, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.28.ffn_up.weight", "offset": 27072578560, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.28.ffn_norm.weight", "offset": 27322139648, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 27322172416, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 27331085312, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.28.attn_q.weight", "offset": 27402388480, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.28.attn_v.weight", "offset": 27473691648, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.29.ffn_gate.weight", "offset": 27482604544, "shape": [ 8192, 28672 ], "size": 8912896, "type": 8 }, { "name": "blk.29.attn_k.weight", "offset": 27732165632, "shape": [ 8192, 1024 ], "size": 249561088, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 27741078528, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.29.attn_q.weight", "offset": 27812381696, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.29.attn_v.weight", "offset": 27883684864, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.29.attn_norm.weight", "offset": 27892597760, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 27892630528, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.29.ffn_up.weight", "offset": 28142191616, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.29.ffn_norm.weight", "offset": 28391752704, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.30.attn_norm.weight", "offset": 28391785472, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 28391818240, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.30.ffn_gate.weight", "offset": 28641379328, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.30.ffn_up.weight", "offset": 28890940416, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.30.ffn_norm.weight", "offset": 29140501504, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 29140534272, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 29149447168, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.30.attn_q.weight", "offset": 29220750336, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.30.attn_v.weight", "offset": 29292053504, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.31.attn_norm.weight", "offset": 29300966400, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 29300999168, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.31.ffn_gate.weight", "offset": 29550560256, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.31.ffn_up.weight", "offset": 29800121344, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.31.ffn_norm.weight", "offset": 30049682432, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 30049715200, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 30058628096, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.31.attn_q.weight", "offset": 30129931264, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.31.attn_v.weight", "offset": 30201234432, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.32.attn_k.weight", "offset": 30210147328, "shape": [ 8192, 1024 ], "size": 8912896, "type": 8 }, { "name": "blk.32.attn_output.weight", "offset": 30219060224, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.32.attn_q.weight", "offset": 30290363392, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.32.attn_v.weight", "offset": 30361666560, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.32.attn_norm.weight", "offset": 30370579456, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 30370612224, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.32.ffn_gate.weight", "offset": 30620173312, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.32.ffn_up.weight", "offset": 30869734400, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.32.ffn_norm.weight", "offset": 31119295488, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 31119328256, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 31119361024, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.33.ffn_gate.weight", "offset": 31368922112, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.33.ffn_up.weight", "offset": 31618483200, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.33.ffn_norm.weight", "offset": 31868044288, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 31868077056, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.33.attn_output.weight", "offset": 31876989952, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.33.attn_q.weight", "offset": 31948293120, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.33.attn_v.weight", "offset": 32019596288, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.34.attn_norm.weight", "offset": 32028509184, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 32028541952, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.34.ffn_gate.weight", "offset": 32278103040, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.34.ffn_up.weight", "offset": 32527664128, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.34.ffn_norm.weight", "offset": 32777225216, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 32777257984, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.34.attn_output.weight", "offset": 32786170880, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.34.attn_q.weight", "offset": 32857474048, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.34.attn_v.weight", "offset": 32928777216, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.35.attn_k.weight", "offset": 32937690112, "shape": [ 8192, 1024 ], "size": 8912896, "type": 8 }, { "name": "blk.35.attn_q.weight", "offset": 32946603008, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.35.attn_v.weight", "offset": 33017906176, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.35.attn_norm.weight", "offset": 33026819072, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 33026851840, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.35.ffn_gate.weight", "offset": 33276412928, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.35.ffn_up.weight", "offset": 33525974016, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.35.ffn_norm.weight", "offset": 33775535104, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.35.attn_output.weight", "offset": 33775567872, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.36.attn_norm.weight", "offset": 33846871040, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 33846903808, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.36.ffn_gate.weight", "offset": 34096464896, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.36.ffn_up.weight", "offset": 34346025984, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.36.ffn_norm.weight", "offset": 34595587072, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 34595619840, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.36.attn_output.weight", "offset": 34604532736, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.36.attn_q.weight", "offset": 34675835904, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.36.attn_v.weight", "offset": 34747139072, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.37.attn_norm.weight", "offset": 34756051968, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 34756084736, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.37.ffn_gate.weight", "offset": 35005645824, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.37.ffn_up.weight", "offset": 35255206912, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.37.ffn_norm.weight", "offset": 35504768000, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 35504800768, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.37.attn_output.weight", "offset": 35513713664, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.37.attn_q.weight", "offset": 35585016832, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.37.attn_v.weight", "offset": 35656320000, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.38.attn_norm.weight", "offset": 35665232896, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 35665265664, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.38.ffn_gate.weight", "offset": 35914826752, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.38.ffn_up.weight", "offset": 36164387840, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.38.ffn_norm.weight", "offset": 36413948928, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 36413981696, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.38.attn_output.weight", "offset": 36422894592, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.38.attn_q.weight", "offset": 36494197760, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.38.attn_v.weight", "offset": 36565500928, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.39.attn_norm.weight", "offset": 36574413824, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 36574446592, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.39.ffn_gate.weight", "offset": 36824007680, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.39.ffn_up.weight", "offset": 37073568768, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.39.ffn_norm.weight", "offset": 37323129856, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 37323162624, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.39.attn_output.weight", "offset": 37332075520, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.39.attn_q.weight", "offset": 37403378688, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.39.attn_v.weight", "offset": 37474681856, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.40.ffn_gate.weight", "offset": 37483594752, "shape": [ 8192, 28672 ], "size": 8912896, "type": 8 }, { "name": "blk.40.ffn_up.weight", "offset": 37733155840, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.40.attn_k.weight", "offset": 37982716928, "shape": [ 8192, 1024 ], "size": 249561088, "type": 8 }, { "name": "blk.40.attn_output.weight", "offset": 37991629824, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.40.attn_q.weight", "offset": 38062932992, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.40.attn_v.weight", "offset": 38134236160, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.40.attn_norm.weight", "offset": 38143149056, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 38143181824, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.40.ffn_norm.weight", "offset": 38392742912, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.41.attn_norm.weight", "offset": 38392775680, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 38392808448, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.41.ffn_gate.weight", "offset": 38642369536, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.41.ffn_up.weight", "offset": 38891930624, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.41.ffn_norm.weight", "offset": 39141491712, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 39141524480, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.41.attn_output.weight", "offset": 39150437376, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.41.attn_q.weight", "offset": 39221740544, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.41.attn_v.weight", "offset": 39293043712, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.42.attn_norm.weight", "offset": 39301956608, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 39301989376, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.42.ffn_gate.weight", "offset": 39551550464, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.42.ffn_up.weight", "offset": 39801111552, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.42.ffn_norm.weight", "offset": 40050672640, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 40050705408, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.42.attn_output.weight", "offset": 40059618304, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.42.attn_q.weight", "offset": 40130921472, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.42.attn_v.weight", "offset": 40202224640, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.43.ffn_gate.weight", "offset": 40211137536, "shape": [ 8192, 28672 ], "size": 8912896, "type": 8 }, { "name": "blk.43.attn_k.weight", "offset": 40460698624, "shape": [ 8192, 1024 ], "size": 249561088, "type": 8 }, { "name": "blk.43.attn_output.weight", "offset": 40469611520, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.43.attn_q.weight", "offset": 40540914688, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.43.attn_v.weight", "offset": 40612217856, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.43.attn_norm.weight", "offset": 40621130752, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 40621163520, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.43.ffn_up.weight", "offset": 40870724608, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.43.ffn_norm.weight", "offset": 41120285696, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 41120318464, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 41120351232, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.44.ffn_gate.weight", "offset": 41369912320, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.44.ffn_up.weight", "offset": 41619473408, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.44.ffn_norm.weight", "offset": 41869034496, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 41869067264, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.44.attn_output.weight", "offset": 41877980160, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.44.attn_q.weight", "offset": 41949283328, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.44.attn_v.weight", "offset": 42020586496, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.45.attn_norm.weight", "offset": 42029499392, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 42029532160, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.45.ffn_gate.weight", "offset": 42279093248, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.45.ffn_up.weight", "offset": 42528654336, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.45.ffn_norm.weight", "offset": 42778215424, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 42778248192, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.45.attn_output.weight", "offset": 42787161088, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.45.attn_q.weight", "offset": 42858464256, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.45.attn_v.weight", "offset": 42929767424, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.46.attn_k.weight", "offset": 42938680320, "shape": [ 8192, 1024 ], "size": 8912896, "type": 8 }, { "name": "blk.46.attn_output.weight", "offset": 42947593216, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.46.attn_q.weight", "offset": 43018896384, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.46.attn_v.weight", "offset": 43090199552, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.46.attn_norm.weight", "offset": 43099112448, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 43099145216, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.46.ffn_gate.weight", "offset": 43348706304, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.46.ffn_up.weight", "offset": 43598267392, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.46.ffn_norm.weight", "offset": 43847828480, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.47.attn_norm.weight", "offset": 43847861248, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 43847894016, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.47.ffn_gate.weight", "offset": 44097455104, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.47.ffn_up.weight", "offset": 44347016192, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.47.ffn_norm.weight", "offset": 44596577280, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 44596610048, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.47.attn_output.weight", "offset": 44605522944, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.47.attn_q.weight", "offset": 44676826112, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.47.attn_v.weight", "offset": 44748129280, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.48.attn_norm.weight", "offset": 44757042176, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 44757074944, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.48.ffn_gate.weight", "offset": 45006636032, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.48.ffn_up.weight", "offset": 45256197120, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.48.ffn_norm.weight", "offset": 45505758208, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 45505790976, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.48.attn_output.weight", "offset": 45514703872, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.48.attn_q.weight", "offset": 45586007040, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.48.attn_v.weight", "offset": 45657310208, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.49.attn_k.weight", "offset": 45666223104, "shape": [ 8192, 1024 ], "size": 8912896, "type": 8 }, { "name": "blk.49.attn_q.weight", "offset": 45675136000, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.49.attn_v.weight", "offset": 45746439168, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.49.attn_norm.weight", "offset": 45755352064, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 45755384832, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.49.ffn_gate.weight", "offset": 46004945920, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.49.ffn_up.weight", "offset": 46254507008, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.49.ffn_norm.weight", "offset": 46504068096, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.49.attn_output.weight", "offset": 46504100864, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.50.attn_norm.weight", "offset": 46575404032, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 46575436800, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.50.ffn_gate.weight", "offset": 46824997888, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.50.ffn_up.weight", "offset": 47074558976, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.50.ffn_norm.weight", "offset": 47324120064, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 47324152832, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.50.attn_output.weight", "offset": 47333065728, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.50.attn_q.weight", "offset": 47404368896, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.50.attn_v.weight", "offset": 47475672064, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.51.attn_norm.weight", "offset": 47484584960, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 47484617728, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.51.ffn_gate.weight", "offset": 47734178816, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.51.ffn_up.weight", "offset": 47983739904, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.51.ffn_norm.weight", "offset": 48233300992, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 48233333760, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.51.attn_output.weight", "offset": 48242246656, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.51.attn_q.weight", "offset": 48313549824, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.51.attn_v.weight", "offset": 48384852992, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.52.attn_norm.weight", "offset": 48393765888, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 48393798656, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.52.ffn_gate.weight", "offset": 48643359744, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.52.ffn_up.weight", "offset": 48892920832, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.52.ffn_norm.weight", "offset": 49142481920, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 49142514688, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.52.attn_output.weight", "offset": 49151427584, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.52.attn_q.weight", "offset": 49222730752, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.52.attn_v.weight", "offset": 49294033920, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.53.attn_norm.weight", "offset": 49302946816, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 49302979584, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.53.ffn_gate.weight", "offset": 49552540672, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.53.ffn_up.weight", "offset": 49802101760, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.53.ffn_norm.weight", "offset": 50051662848, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 50051695616, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.53.attn_output.weight", "offset": 50060608512, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.53.attn_q.weight", "offset": 50131911680, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.53.attn_v.weight", "offset": 50203214848, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.54.ffn_gate.weight", "offset": 50212127744, "shape": [ 8192, 28672 ], "size": 8912896, "type": 8 }, { "name": "blk.54.ffn_up.weight", "offset": 50461688832, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.54.attn_k.weight", "offset": 50711249920, "shape": [ 8192, 1024 ], "size": 249561088, "type": 8 }, { "name": "blk.54.attn_output.weight", "offset": 50720162816, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.54.attn_q.weight", "offset": 50791465984, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.54.attn_v.weight", "offset": 50862769152, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.54.attn_norm.weight", "offset": 50871682048, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 50871714816, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.54.ffn_norm.weight", "offset": 51121275904, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.55.attn_norm.weight", "offset": 51121308672, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 51121341440, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.55.ffn_gate.weight", "offset": 51370902528, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.55.ffn_up.weight", "offset": 51620463616, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.55.ffn_norm.weight", "offset": 51870024704, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 51870057472, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.55.attn_output.weight", "offset": 51878970368, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.55.attn_q.weight", "offset": 51950273536, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.55.attn_v.weight", "offset": 52021576704, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.56.attn_norm.weight", "offset": 52030489600, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 52030522368, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.56.ffn_gate.weight", "offset": 52280083456, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.56.ffn_up.weight", "offset": 52529644544, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.56.ffn_norm.weight", "offset": 52779205632, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 52779238400, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.56.attn_output.weight", "offset": 52788151296, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.56.attn_q.weight", "offset": 52859454464, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.56.attn_v.weight", "offset": 52930757632, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.57.ffn_gate.weight", "offset": 52939670528, "shape": [ 8192, 28672 ], "size": 8912896, "type": 8 }, { "name": "blk.57.attn_k.weight", "offset": 53189231616, "shape": [ 8192, 1024 ], "size": 249561088, "type": 8 }, { "name": "blk.57.attn_output.weight", "offset": 53198144512, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.57.attn_q.weight", "offset": 53269447680, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.57.attn_v.weight", "offset": 53340750848, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.57.attn_norm.weight", "offset": 53349663744, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 53349696512, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.57.ffn_up.weight", "offset": 53599257600, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.57.ffn_norm.weight", "offset": 53848818688, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.58.attn_norm.weight", "offset": 53848851456, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 53848884224, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.58.ffn_gate.weight", "offset": 54098445312, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.58.ffn_up.weight", "offset": 54348006400, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.58.ffn_norm.weight", "offset": 54597567488, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 54597600256, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.58.attn_output.weight", "offset": 54606513152, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.58.attn_q.weight", "offset": 54677816320, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.58.attn_v.weight", "offset": 54749119488, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.59.attn_norm.weight", "offset": 54758032384, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 54758065152, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.59.ffn_gate.weight", "offset": 55007626240, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.59.ffn_up.weight", "offset": 55257187328, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.59.ffn_norm.weight", "offset": 55506748416, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 55506781184, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.59.attn_output.weight", "offset": 55515694080, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.59.attn_q.weight", "offset": 55586997248, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.59.attn_v.weight", "offset": 55658300416, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.60.attn_k.weight", "offset": 55667213312, "shape": [ 8192, 1024 ], "size": 8912896, "type": 8 }, { "name": "blk.60.attn_output.weight", "offset": 55676126208, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.60.attn_q.weight", "offset": 55747429376, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.60.attn_v.weight", "offset": 55818732544, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.60.attn_norm.weight", "offset": 55827645440, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 55827678208, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.60.ffn_gate.weight", "offset": 56077239296, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.60.ffn_up.weight", "offset": 56326800384, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.60.ffn_norm.weight", "offset": 56576361472, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.61.attn_norm.weight", "offset": 56576394240, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 56576427008, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.61.ffn_gate.weight", "offset": 56825988096, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.61.ffn_up.weight", "offset": 57075549184, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.61.ffn_norm.weight", "offset": 57325110272, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 57325143040, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.61.attn_output.weight", "offset": 57334055936, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.61.attn_q.weight", "offset": 57405359104, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.61.attn_v.weight", "offset": 57476662272, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.62.attn_norm.weight", "offset": 57485575168, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 57485607936, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.62.ffn_gate.weight", "offset": 57735169024, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.62.ffn_up.weight", "offset": 57984730112, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.62.ffn_norm.weight", "offset": 58234291200, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 58234323968, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.62.attn_output.weight", "offset": 58243236864, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.62.attn_q.weight", "offset": 58314540032, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.62.attn_v.weight", "offset": 58385843200, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.63.attn_k.weight", "offset": 58394756096, "shape": [ 8192, 1024 ], "size": 8912896, "type": 8 }, { "name": "blk.63.attn_q.weight", "offset": 58403668992, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.63.attn_v.weight", "offset": 58474972160, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.63.attn_norm.weight", "offset": 58483885056, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 58483917824, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.63.ffn_gate.weight", "offset": 58733478912, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.63.ffn_up.weight", "offset": 58983040000, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.63.ffn_norm.weight", "offset": 59232601088, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.63.attn_output.weight", "offset": 59232633856, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.64.attn_norm.weight", "offset": 59303937024, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 59303969792, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.64.ffn_gate.weight", "offset": 59553530880, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.64.ffn_up.weight", "offset": 59803091968, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.64.ffn_norm.weight", "offset": 60052653056, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 60052685824, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.64.attn_output.weight", "offset": 60061598720, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.64.attn_q.weight", "offset": 60132901888, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.64.attn_v.weight", "offset": 60204205056, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.65.attn_norm.weight", "offset": 60213117952, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 60213150720, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.65.ffn_gate.weight", "offset": 60462711808, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.65.ffn_up.weight", "offset": 60712272896, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.65.ffn_norm.weight", "offset": 60961833984, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 60961866752, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.65.attn_output.weight", "offset": 60970779648, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.65.attn_q.weight", "offset": 61042082816, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.65.attn_v.weight", "offset": 61113385984, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.66.attn_norm.weight", "offset": 61122298880, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 61122331648, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.66.ffn_gate.weight", "offset": 61371892736, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.66.ffn_up.weight", "offset": 61621453824, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.66.ffn_norm.weight", "offset": 61871014912, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 61871047680, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.66.attn_output.weight", "offset": 61879960576, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.66.attn_q.weight", "offset": 61951263744, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.66.attn_v.weight", "offset": 62022566912, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.67.attn_norm.weight", "offset": 62031479808, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 62031512576, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.67.ffn_gate.weight", "offset": 62281073664, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.67.ffn_up.weight", "offset": 62530634752, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.67.ffn_norm.weight", "offset": 62780195840, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 62780228608, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.67.attn_output.weight", "offset": 62789141504, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.67.attn_q.weight", "offset": 62860444672, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.67.attn_v.weight", "offset": 62931747840, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.68.ffn_gate.weight", "offset": 62940660736, "shape": [ 8192, 28672 ], "size": 8912896, "type": 8 }, { "name": "blk.68.ffn_up.weight", "offset": 63190221824, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.68.attn_k.weight", "offset": 63439782912, "shape": [ 8192, 1024 ], "size": 249561088, "type": 8 }, { "name": "blk.68.attn_output.weight", "offset": 63448695808, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.68.attn_q.weight", "offset": 63519998976, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.68.attn_v.weight", "offset": 63591302144, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.68.attn_norm.weight", "offset": 63600215040, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.68.ffn_down.weight", "offset": 63600247808, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.68.ffn_norm.weight", "offset": 63849808896, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 63849841664, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 63849874432, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.69.ffn_gate.weight", "offset": 64099435520, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.69.ffn_up.weight", "offset": 64348996608, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.69.ffn_norm.weight", "offset": 64598557696, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 64598590464, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.69.attn_output.weight", "offset": 64607503360, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.69.attn_q.weight", "offset": 64678806528, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.69.attn_v.weight", "offset": 64750109696, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.70.attn_norm.weight", "offset": 64759022592, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 64759055360, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.70.ffn_gate.weight", "offset": 65008616448, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.70.ffn_up.weight", "offset": 65258177536, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.70.ffn_norm.weight", "offset": 65507738624, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 65507771392, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.70.attn_output.weight", "offset": 65516684288, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.70.attn_q.weight", "offset": 65587987456, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.70.attn_v.weight", "offset": 65659290624, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.71.ffn_gate.weight", "offset": 65668203520, "shape": [ 8192, 28672 ], "size": 8912896, "type": 8 }, { "name": "blk.71.attn_k.weight", "offset": 65917764608, "shape": [ 8192, 1024 ], "size": 249561088, "type": 8 }, { "name": "blk.71.attn_output.weight", "offset": 65926677504, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.71.attn_q.weight", "offset": 65997980672, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.71.attn_v.weight", "offset": 66069283840, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.71.attn_norm.weight", "offset": 66078196736, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 66078229504, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.71.ffn_up.weight", "offset": 66327790592, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.71.ffn_norm.weight", "offset": 66577351680, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.72.attn_norm.weight", "offset": 66577384448, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 66577417216, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.72.ffn_gate.weight", "offset": 66826978304, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.72.ffn_up.weight", "offset": 67076539392, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.72.ffn_norm.weight", "offset": 67326100480, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 67326133248, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.72.attn_output.weight", "offset": 67335046144, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.72.attn_q.weight", "offset": 67406349312, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.72.attn_v.weight", "offset": 67477652480, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.73.attn_norm.weight", "offset": 67486565376, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 67486598144, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.73.ffn_gate.weight", "offset": 67736159232, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.73.ffn_up.weight", "offset": 67985720320, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.73.ffn_norm.weight", "offset": 68235281408, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 68235314176, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.73.attn_output.weight", "offset": 68244227072, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.73.attn_q.weight", "offset": 68315530240, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.73.attn_v.weight", "offset": 68386833408, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.74.attn_k.weight", "offset": 68395746304, "shape": [ 8192, 1024 ], "size": 8912896, "type": 8 }, { "name": "blk.74.attn_output.weight", "offset": 68404659200, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.74.attn_q.weight", "offset": 68475962368, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.74.attn_v.weight", "offset": 68547265536, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.74.attn_norm.weight", "offset": 68556178432, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 68556211200, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.74.ffn_gate.weight", "offset": 68805772288, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.74.ffn_up.weight", "offset": 69055333376, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.74.ffn_norm.weight", "offset": 69304894464, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.75.attn_norm.weight", "offset": 69304927232, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 69304960000, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.75.ffn_gate.weight", "offset": 69554521088, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.75.ffn_up.weight", "offset": 69804082176, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.75.ffn_norm.weight", "offset": 70053643264, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 70053676032, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.75.attn_output.weight", "offset": 70062588928, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.75.attn_q.weight", "offset": 70133892096, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.75.attn_v.weight", "offset": 70205195264, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.76.attn_norm.weight", "offset": 70214108160, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 70214140928, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.76.ffn_gate.weight", "offset": 70463702016, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.76.ffn_up.weight", "offset": 70713263104, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.76.ffn_norm.weight", "offset": 70962824192, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 70962856960, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.76.attn_output.weight", "offset": 70971769856, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.76.attn_q.weight", "offset": 71043073024, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.76.attn_v.weight", "offset": 71114376192, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.77.attn_k.weight", "offset": 71123289088, "shape": [ 8192, 1024 ], "size": 8912896, "type": 8 }, { "name": "blk.77.attn_q.weight", "offset": 71132201984, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.77.attn_v.weight", "offset": 71203505152, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.77.attn_norm.weight", "offset": 71212418048, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 71212450816, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.77.ffn_gate.weight", "offset": 71462011904, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.77.ffn_up.weight", "offset": 71711572992, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.77.ffn_norm.weight", "offset": 71961134080, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.77.attn_output.weight", "offset": 71961166848, "shape": [ 8192, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.78.attn_norm.weight", "offset": 72032470016, "shape": [ 8192 ], "size": 71303168, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 72032502784, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.78.ffn_gate.weight", "offset": 72282063872, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.78.ffn_up.weight", "offset": 72531624960, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.78.ffn_norm.weight", "offset": 72781186048, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 72781218816, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.78.attn_output.weight", "offset": 72790131712, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.78.attn_q.weight", "offset": 72861434880, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.78.attn_v.weight", "offset": 72932738048, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "blk.79.attn_norm.weight", "offset": 72941650944, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 72941683712, "shape": [ 28672, 8192 ], "size": 32768, "type": 8 }, { "name": "blk.79.ffn_gate.weight", "offset": 73191244800, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.79.ffn_up.weight", "offset": 73440805888, "shape": [ 8192, 28672 ], "size": 249561088, "type": 8 }, { "name": "blk.79.ffn_norm.weight", "offset": 73690366976, "shape": [ 8192 ], "size": 249561088, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 73690399744, "shape": [ 8192, 1024 ], "size": 32768, "type": 8 }, { "name": "blk.79.attn_output.weight", "offset": 73699312640, "shape": [ 8192, 8192 ], "size": 8912896, "type": 8 }, { "name": "blk.79.attn_q.weight", "offset": 73770615808, "shape": [ 8192, 8192 ], "size": 71303168, "type": 8 }, { "name": "blk.79.attn_v.weight", "offset": 73841918976, "shape": [ 8192, 1024 ], "size": 71303168, "type": 8 }, { "name": "output_norm.weight", "offset": 73850831872, "shape": [ 8192 ], "size": 8912896, "type": 0 }, { "name": "output.weight", "offset": 73850864640, "shape": [ 8192, 128258 ], "size": 32768, "type": 8 } ], "version": 3 }