qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 1, "general.name": "Qwen2-7B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 1089994752, "shape": [ 3584 ], "size": 1089994752, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 1090009088, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.0.ffn_gate.weight", "offset": 1225799680, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.0.ffn_up.weight", "offset": 1361590272, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.0.ffn_norm.weight", "offset": 1497380864, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 1497395200, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 1497397248, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.0.attn_output.weight", "offset": 1501067264, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.0.attn_q.bias", "offset": 1526757376, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 1526771712, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.0.attn_v.bias", "offset": 1552461824, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 1552463872, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.1.attn_norm.weight", "offset": 1556133888, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1556148224, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.1.ffn_gate.weight", "offset": 1691938816, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.1.ffn_up.weight", "offset": 1827729408, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.1.ffn_norm.weight", "offset": 1963520000, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 1963534336, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1963536384, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.1.attn_output.weight", "offset": 1967206400, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.1.attn_q.bias", "offset": 1992896512, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 1992910848, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.1.attn_v.bias", "offset": 2018600960, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 2018603008, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 2022273024, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 2022287360, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.2.ffn_gate.weight", "offset": 2158077952, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.2.ffn_up.weight", "offset": 2293868544, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.2.ffn_norm.weight", "offset": 2429659136, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 2429673472, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 2429675520, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.2.attn_output.weight", "offset": 2433345536, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.2.attn_q.bias", "offset": 2459035648, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 2459049984, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.2.attn_v.bias", "offset": 2484740096, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 2484742144, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.3.attn_norm.weight", "offset": 2488412160, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 2488426496, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.3.ffn_gate.weight", "offset": 2624217088, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.3.ffn_up.weight", "offset": 2760007680, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.3.ffn_norm.weight", "offset": 2895798272, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 2895812608, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 2895814656, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.3.attn_output.weight", "offset": 2899484672, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.3.attn_q.bias", "offset": 2925174784, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 2925189120, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.3.attn_v.bias", "offset": 2950879232, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 2950881280, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 2954551296, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 2954565632, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.4.ffn_gate.weight", "offset": 3090356224, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.4.ffn_up.weight", "offset": 3226146816, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.4.ffn_norm.weight", "offset": 3361937408, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 3361951744, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 3361953792, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.4.attn_output.weight", "offset": 3365623808, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.4.attn_q.bias", "offset": 3391313920, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 3391328256, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "\u0000\u0000\u0000blk.16.attn_v.", "offset": 3417018368, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 3417020416, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.5.attn_norm.weight", "offset": 3420690432, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 3420704768, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.5.ffn_gate.weight", "offset": 3556495360, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.5.ffn_up.weight", "offset": 3692285952, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.5.ffn_norm.weight", "offset": 3828076544, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 3828090880, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 3828092928, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.5.attn_output.weight", "offset": 3831762944, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.5.attn_q.bias", "offset": 3857453056, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 3857467392, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.5.attn_v.bias", "offset": 3883157504, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 3883159552, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.6.attn_k.bias", "offset": 3886829568, "shape": [ 512 ], "size": 3670016, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 3886831616, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.6.attn_output.weight", "offset": 3890501632, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.6.attn_q.bias", "offset": 3916191744, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 3916206080, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.6.attn_v.bias", "offset": 3941896192, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 3941898240, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.10.attn_norm.weight", "offset": 3945568256, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 3945582592, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.10.ffn_gate.weight", "offset": 4081373184, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.10.ffn_up.weight", "offset": 4217163776, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.10.ffn_norm.weight", "offset": 4352954368, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 4352968704, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 4352970752, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.10.attn_output.weight", "offset": 4356640768, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.10.attn_q.bias", "offset": 4382330880, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 4382345216, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.10.attn_v.bias", "offset": 4408035328, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 4408037376, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.11.attn_norm.weight", "offset": 4411707392, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 4411721728, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.11.ffn_gate.weight", "offset": 4547512320, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.11.ffn_up.weight", "offset": 4683302912, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.11.ffn_norm.weight", "offset": 4819093504, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 4819107840, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 4819109888, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.11.attn_output.weight", "offset": 4822779904, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.11.attn_q.bias", "offset": 4848470016, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 4848484352, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.11.attn_v.bias", "offset": 4874174464, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 4874176512, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 4877846528, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 4877860864, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.12.ffn_gate.weight", "offset": 5013651456, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.12.ffn_up.weight", "offset": 5149442048, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.12.ffn_norm.weight", "offset": 5285232640, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 5285246976, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 5285249024, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.12.attn_output.weight", "offset": 5288919040, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.12.attn_q.bias", "offset": 5314609152, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 5314623488, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.12.attn_v.bias", "offset": 5340313600, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 5340315648, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.13.attn_norm.weight", "offset": 5343985664, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 5344000000, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.13.ffn_gate.weight", "offset": 5479790592, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.13.ffn_up.weight", "offset": 5615581184, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.13.ffn_norm.weight", "offset": 5751371776, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 5751386112, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 5751388160, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.13.attn_output.weight", "offset": 5755058176, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.13.attn_q.bias", "offset": 5780748288, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 5780762624, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.13.attn_v.bias", "offset": 5806452736, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 5806454784, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.14.ffn_gate.weight", "offset": 5810124800, "shape": [ 3584, 18944 ], "size": 3670016, "type": 1 }, { "name": "blk.14.attn_k.bias", "offset": 5945915392, "shape": [ 512 ], "size": 135790592, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 5945917440, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.14.attn_output.weight", "offset": 5949587456, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.14.attn_q.bias", "offset": 5975277568, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 5975291904, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.14.attn_v.bias", "offset": 6000982016, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 6000984064, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.6.attn_norm.weight", "offset": 6004654080, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 6004668416, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.6.ffn_gate.weight", "offset": 6140459008, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.6.ffn_up.weight", "offset": 6276249600, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.6.ffn_norm.weight", "offset": 6412040192, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 6412054528, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 6412068864, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.7.ffn_gate.weight", "offset": 6547859456, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.7.ffn_up.weight", "offset": 6683650048, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.7.ffn_norm.weight", "offset": 6819440640, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 6819454976, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 6819457024, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.7.attn_output.weight", "offset": 6823127040, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.7.attn_q.bias", "offset": 6848817152, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 6848831488, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.7.attn_v.bias", "offset": 6874521600, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 6874523648, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.8.attn_norm.weight", "offset": 6878193664, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 6878208000, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.8.ffn_gate.weight", "offset": 7013998592, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.8.ffn_up.weight", "offset": 7149789184, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.8.ffn_norm.weight", "offset": 7285579776, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 7285594112, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 7285596160, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.8.attn_output.weight", "offset": 7289266176, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.8.attn_q.bias", "offset": 7314956288, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 7314970624, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.8.attn_v.bias", "offset": 7340660736, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 7340662784, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 7344332800, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 7344347136, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.9.ffn_gate.weight", "offset": 7480137728, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.9.ffn_up.weight", "offset": 7615928320, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.9.ffn_norm.weight", "offset": 7751718912, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 7751733248, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 7751735296, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.9.attn_output.weight", "offset": 7755405312, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.9.attn_q.bias", "offset": 7781095424, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 7781109760, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.9.attn_v.bias", "offset": 7806799872, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 7806801920, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 7810471936, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 7810486272, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.14.ffn_up.weight", "offset": 7946276864, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.14.ffn_norm.weight", "offset": 8082067456, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 8082081792, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 8082096128, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.15.ffn_gate.weight", "offset": 8217886720, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.15.ffn_up.weight", "offset": 8353677312, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.15.ffn_norm.weight", "offset": 8489467904, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 8489482240, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 8489484288, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.15.attn_output.weight", "offset": 8493154304, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.15.attn_q.bias", "offset": 8518844416, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 8518858752, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.15.attn_v.bias", "offset": 8544548864, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 8544550912, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.16.attn_norm.weight", "offset": 8548220928, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 8548235264, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.16.ffn_gate.weight", "offset": 8684025856, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.16.ffn_up.weight", "offset": 8819816448, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.16.ffn_norm.weight", "offset": 8955607040, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 8955621376, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 8955623424, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.16.attn_output.weight", "offset": 8959293440, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.16.attn_q.bias", "offset": 8984983552, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 8984997888, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.16.attn_v.bias", "offset": 9010688000, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 9010690048, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 9014360064, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 9014374400, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.17.ffn_gate.weight", "offset": 9150164992, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.17.ffn_up.weight", "offset": 9285955584, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.17.ffn_norm.weight", "offset": 9421746176, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 9421760512, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 9421762560, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.17.attn_output.weight", "offset": 9425432576, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.17.attn_q.bias", "offset": 9451122688, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 9451137024, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.17.attn_v.bias", "offset": 9476827136, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 9476829184, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.18.attn_norm.weight", "offset": 9480499200, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 9480513536, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.18.ffn_gate.weight", "offset": 9616304128, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.18.ffn_up.weight", "offset": 9752094720, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.18.ffn_norm.weight", "offset": 9887885312, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 9887899648, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 9887901696, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.18.attn_output.weight", "offset": 9891571712, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.18.attn_q.bias", "offset": 9917261824, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 9917276160, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.18.attn_v.bias", "offset": 9942966272, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 9942968320, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.19.attn_norm.weight", "offset": 9946638336, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 9946652672, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.19.ffn_gate.weight", "offset": 10082443264, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.19.ffn_up.weight", "offset": 10218233856, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.19.ffn_norm.weight", "offset": 10354024448, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 10354038784, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 10354040832, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.19.attn_output.weight", "offset": 10357710848, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.19.attn_q.bias", "offset": 10383400960, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 10383415296, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.19.attn_v.bias", "offset": 10409105408, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 10409107456, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.20.attn_norm.weight", "offset": 10412777472, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 10412791808, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.20.ffn_gate.weight", "offset": 10548582400, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.20.ffn_up.weight", "offset": 10684372992, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.20.ffn_norm.weight", "offset": 10820163584, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 10820177920, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 10820179968, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.20.attn_output.weight", "offset": 10823849984, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.20.attn_q.bias", "offset": 10849540096, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 10849554432, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.20.attn_v.bias", "offset": 10875244544, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 10875246592, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 10878916608, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 10878930944, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.21.ffn_gate.weight", "offset": 11014721536, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.21.ffn_up.weight", "offset": 11150512128, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.21.ffn_norm.weight", "offset": 11286302720, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 11286317056, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 11286319104, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.21.attn_output.weight", "offset": 11289989120, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.21.attn_q.bias", "offset": 11315679232, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 11315693568, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.21.attn_v.bias", "offset": 11341383680, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 11341385728, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.22.ffn_gate.weight", "offset": 11345055744, "shape": [ 3584, 18944 ], "size": 3670016, "type": 1 }, { "name": "blk.22.ffn_up.weight", "offset": 11480846336, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.22.attn_k.bias", "offset": 11616636928, "shape": [ 512 ], "size": 135790592, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 11616638976, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.22.attn_output.weight", "offset": 11620308992, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.22.attn_q.bias", "offset": 11645999104, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 11646013440, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.22.attn_v.bias", "offset": 11671703552, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 11671705600, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "output.weight", "offset": 11675375616, "shape": [ 3584, 152064 ], "size": 3670016, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 12765370368, "shape": [ 3584 ], "size": 1089994752, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 12765384704, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.22.ffn_norm.weight", "offset": 12901175296, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 12901189632, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 12901203968, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.23.ffn_gate.weight", "offset": 13036994560, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.23.ffn_up.weight", "offset": 13172785152, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.23.ffn_norm.weight", "offset": 13308575744, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 13308590080, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 13308592128, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.23.attn_output.weight", "offset": 13312262144, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.23.attn_q.bias", "offset": 13337952256, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 13337966592, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.23.attn_v.bias", "offset": 13363656704, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 13363658752, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 13367328768, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 13367343104, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.24.ffn_gate.weight", "offset": 13503133696, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.24.ffn_up.weight", "offset": 13638924288, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.24.ffn_norm.weight", "offset": 13774714880, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 13774729216, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 13774731264, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.24.attn_output.weight", "offset": 13778401280, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.24.attn_q.bias", "offset": 13804091392, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 13804105728, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.24.attn_v.bias", "offset": 13829795840, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 13829797888, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.25.attn_norm.weight", "offset": 13833467904, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 13833482240, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.25.ffn_gate.weight", "offset": 13969272832, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.25.ffn_up.weight", "offset": 14105063424, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.25.ffn_norm.weight", "offset": 14240854016, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 14240868352, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 14240870400, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.25.attn_output.weight", "offset": 14244540416, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.25.attn_q.bias", "offset": 14270230528, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 14270244864, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.25.attn_v.bias", "offset": 14295934976, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 14295937024, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 14299607040, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 14299621376, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.26.ffn_gate.weight", "offset": 14435411968, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.26.ffn_up.weight", "offset": 14571202560, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.26.ffn_norm.weight", "offset": 14706993152, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 14707007488, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 14707009536, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.26.attn_output.weight", "offset": 14710679552, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.26.attn_q.bias", "offset": 14736369664, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 14736384000, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.26.attn_v.bias", "offset": 14762074112, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 14762076160, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.27.attn_norm.weight", "offset": 14765746176, "shape": [ 3584 ], "size": 3670016, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 14765760512, "shape": [ 18944, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.27.ffn_gate.weight", "offset": 14901551104, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.27.ffn_up.weight", "offset": 15037341696, "shape": [ 3584, 18944 ], "size": 135790592, "type": 1 }, { "name": "blk.27.ffn_norm.weight", "offset": 15173132288, "shape": [ 3584 ], "size": 135790592, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 15173146624, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 15173148672, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "blk.27.attn_output.weight", "offset": 15176818688, "shape": [ 3584, 3584 ], "size": 3670016, "type": 1 }, { "name": "blk.27.attn_q.bias", "offset": 15202508800, "shape": [ 3584 ], "size": 25690112, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 15202523136, "shape": [ 3584, 3584 ], "size": 14336, "type": 1 }, { "name": "blk.27.attn_v.bias", "offset": 15228213248, "shape": [ 512 ], "size": 25690112, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 15228215296, "shape": [ 3584, 512 ], "size": 2048, "type": 1 }, { "name": "output_norm.weight", "offset": 15231885312, "shape": [ 3584 ], "size": 3670016, "type": 0 } ], "version": 3 }