qwen2

Qwen2 is a new series of large language models from Alibaba group

0.5B 1.5B 7B 72B

273.2K Pulls Updated 4 weeks ago

97 Tags

... /
model
3fbb0c6fe2e7 · 397MB
{ "metadata": { "general.architecture": "qwen2", "general.file_type": 8, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.0.ffn_gate.weight", "offset": 147642880, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.0.ffn_up.weight", "offset": 150639104, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.0.ffn_norm.weight", "offset": 153635328, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 153638912, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 153639424, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.0.attn_output.weight", "offset": 153718272, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.0.attn_q.bias", "offset": 154270208, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 154273792, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.0.attn_v.bias", "offset": 154825728, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 154826240, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.1.attn_norm.weight", "offset": 154905088, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 154908672, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.1.ffn_gate.weight", "offset": 157904896, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.1.ffn_up.weight", "offset": 160901120, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.1.ffn_norm.weight", "offset": 163897344, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 163900928, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 163901440, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.1.attn_output.weight", "offset": 163980288, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.1.attn_q.bias", "offset": 164532224, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 164535808, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.1.attn_v.bias", "offset": 165087744, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 165088256, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.10.attn_norm.weight", "offset": 165167104, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 165170688, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.10.ffn_gate.weight", "offset": 168166912, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.10.ffn_up.weight", "offset": 171163136, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.10.ffn_norm.weight", "offset": 174159360, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 174162944, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 174163456, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.10.attn_output.weight", "offset": 174242304, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.10.attn_q.bias", "offset": 174794240, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 174797824, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.10.attn_v.bias", "offset": 175349760, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 175350272, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.11.attn_norm.weight", "offset": 175429120, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 175432704, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "\u0000\u0000\u0000\ufffd\u0007\u000b\u0000\u0000\u0000\u0000\u0014\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.", "offset": 178428928, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.11.ffn_up.weight", "offset": 181425152, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.11.ffn_norm.weight", "offset": 184421376, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 184424960, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 184425472, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.11.attn_output.weight", "offset": 184504320, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.11.attn_q.bias", "offset": 185056256, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 185059840, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.11.attn_v.bias", "offset": 185611776, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 185612288, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.12.attn_norm.weight", "offset": 185691136, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 185694720, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.12.ffn_gate.weight", "offset": 188690944, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.12.ffn_up.weight", "offset": 191687168, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.12.ffn_norm.weight", "offset": 194683392, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 194686976, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 194687488, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.12.attn_output.weight", "offset": 194766336, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.12.attn_q.bias", "offset": 195318272, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 195321856, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.12.attn_v.bias", "offset": 195873792, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 195874304, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.13.attn_norm.weight", "offset": 195953152, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 195956736, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.13.ffn_gate.weight", "offset": 198952960, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.13.ffn_up.weight", "offset": 201949184, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.13.ffn_norm.weight", "offset": 204945408, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 204948992, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 204949504, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.13.attn_output.weight", "offset": 205028352, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.13.attn_q.bias", "offset": 205580288, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 205583872, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.13.attn_v.bias", "offset": 206135808, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 206136320, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.14.attn_norm.weight", "offset": 206215168, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 206218752, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.14.ffn_gate.weight", "offset": 209214976, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.14.ffn_up.weight", "offset": 212211200, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.14.ffn_norm.weight", "offset": 215207424, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 215211008, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 215211520, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.14.attn_output.weight", "offset": 215290368, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.14.attn_q.bias", "offset": 215842304, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 215845888, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.14.attn_v.bias", "offset": 216397824, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 216398336, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.15.attn_norm.weight", "offset": 216477184, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 216480768, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.15.ffn_gate.weight", "offset": 219476992, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.15.ffn_up.weight", "offset": 222473216, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.15.ffn_norm.weight", "offset": 225469440, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 225473024, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 225473536, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.15.attn_output.weight", "offset": 225552384, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.15.attn_q.bias", "offset": 226104320, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 226107904, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.15.attn_v.bias", "offset": 226659840, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 226660352, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.16.attn_norm.weight", "offset": 226739200, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 226742784, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.16.ffn_gate.weight", "offset": 229739008, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.16.ffn_up.weight", "offset": 232735232, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.16.ffn_norm.weight", "offset": 235731456, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 235735040, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 235735552, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.16.attn_output.weight", "offset": 235814400, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.16.attn_q.bias", "offset": 236366336, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 236369920, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.16.attn_v.bias", "offset": 236921856, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 236922368, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.17.attn_norm.weight", "offset": 237001216, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 237004800, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.17.ffn_gate.weight", "offset": 240001024, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.17.ffn_up.weight", "offset": 242997248, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.17.ffn_norm.weight", "offset": 245993472, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 245997056, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 245997568, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.17.attn_output.weight", "offset": 246076416, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.17.attn_q.bias", "offset": 246628352, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 246631936, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.17.attn_v.bias", "offset": 247183872, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 247184384, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.18.attn_norm.weight", "offset": 247263232, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 247266816, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.18.ffn_gate.weight", "offset": 250263040, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.18.ffn_up.weight", "offset": 253259264, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.18.ffn_norm.weight", "offset": 256255488, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 256259072, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 256259584, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.18.attn_output.weight", "offset": 256338432, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.18.attn_q.bias", "offset": 256890368, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 256893952, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.18.attn_v.bias", "offset": 257445888, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 257446400, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.19.attn_norm.weight", "offset": 257525248, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 257528832, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.19.ffn_gate.weight", "offset": 260525056, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.19.ffn_up.weight", "offset": 263521280, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.19.ffn_norm.weight", "offset": 266517504, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 266521088, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 266521600, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.19.attn_output.weight", "offset": 266600448, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.19.attn_q.bias", "offset": 267152384, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 267155968, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.19.attn_v.bias", "offset": 267707904, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 267708416, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.2.attn_norm.weight", "offset": 267787264, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 267790848, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.2.ffn_gate.weight", "offset": 270787072, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.2.ffn_up.weight", "offset": 273783296, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.2.ffn_norm.weight", "offset": 276779520, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 276783104, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 276783616, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.2.attn_output.weight", "offset": 276862464, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.2.attn_q.bias", "offset": 277414400, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 277417984, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.2.attn_v.bias", "offset": 277969920, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 277970432, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.20.attn_norm.weight", "offset": 278049280, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 278052864, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.20.ffn_gate.weight", "offset": 281049088, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.20.ffn_up.weight", "offset": 284045312, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.20.ffn_norm.weight", "offset": 287041536, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 287045120, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 287045632, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.20.attn_output.weight", "offset": 287124480, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.20.attn_q.bias", "offset": 287676416, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 287680000, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.20.attn_v.bias", "offset": 288231936, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 288232448, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.21.attn_norm.weight", "offset": 288311296, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 288314880, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.21.ffn_gate.weight", "offset": 291311104, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.21.ffn_up.weight", "offset": 294307328, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.21.ffn_norm.weight", "offset": 297303552, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 297307136, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 297307648, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.21.attn_output.weight", "offset": 297386496, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.21.attn_q.bias", "offset": 297938432, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 297942016, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.21.attn_v.bias", "offset": 298493952, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 298494464, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.22.attn_norm.weight", "offset": 298573312, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 298576896, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.22.ffn_gate.weight", "offset": 301573120, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.22.ffn_up.weight", "offset": 304569344, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.22.ffn_norm.weight", "offset": 307565568, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 307569152, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 307569664, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.22.attn_output.weight", "offset": 307648512, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.22.attn_q.bias", "offset": 308200448, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 308204032, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.22.attn_v.bias", "offset": 308755968, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 308756480, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.23.attn_norm.weight", "offset": 308835328, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 308838912, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.23.ffn_gate.weight", "offset": 311835136, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.23.ffn_up.weight", "offset": 314831360, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.23.ffn_norm.weight", "offset": 317827584, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 317831168, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 317831680, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.23.attn_output.weight", "offset": 317910528, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.23.attn_q.bias", "offset": 318462464, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 318466048, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.23.attn_v.bias", "offset": 319017984, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 319018496, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.3.attn_norm.weight", "offset": 319097344, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 319100928, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.3.ffn_gate.weight", "offset": 322097152, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.3.ffn_up.weight", "offset": 325093376, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.3.ffn_norm.weight", "offset": 328089600, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 328093184, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 328093696, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.3.attn_output.weight", "offset": 328172544, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.3.attn_q.bias", "offset": 328724480, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 328728064, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.3.attn_v.bias", "offset": 329280000, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 329280512, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.4.attn_norm.weight", "offset": 329359360, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 329362944, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.4.ffn_gate.weight", "offset": 332359168, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.4.ffn_up.weight", "offset": 335355392, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.4.ffn_norm.weight", "offset": 338351616, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 338355200, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 338355712, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.4.attn_output.weight", "offset": 338434560, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.4.attn_q.bias", "offset": 338986496, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 338990080, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.4.attn_v.bias", "offset": 339542016, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 339542528, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.5.attn_norm.weight", "offset": 339621376, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 339624960, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.5.ffn_gate.weight", "offset": 342621184, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.5.ffn_up.weight", "offset": 345617408, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.5.ffn_norm.weight", "offset": 348613632, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 348617216, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 348617728, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.5.attn_output.weight", "offset": 348696576, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.5.attn_q.bias", "offset": 349248512, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 349252096, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.5.attn_v.bias", "offset": 349804032, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 349804544, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.6.attn_norm.weight", "offset": 349883392, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 349886976, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.6.ffn_gate.weight", "offset": 352883200, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.6.ffn_up.weight", "offset": 355879424, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.6.ffn_norm.weight", "offset": 358875648, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 358879232, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 358879744, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.6.attn_output.weight", "offset": 358958592, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.6.attn_q.bias", "offset": 359510528, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 359514112, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.6.attn_v.bias", "offset": 360066048, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 360066560, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.7.attn_norm.weight", "offset": 360145408, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 360148992, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.7.ffn_gate.weight", "offset": 363145216, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.7.ffn_up.weight", "offset": 366141440, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.7.ffn_norm.weight", "offset": 369137664, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 369141248, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 369141760, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.7.attn_output.weight", "offset": 369220608, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.7.attn_q.bias", "offset": 369772544, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 369776128, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.7.attn_v.bias", "offset": 370328064, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 370328576, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.8.attn_norm.weight", "offset": 370407424, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 370411008, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.8.ffn_gate.weight", "offset": 373407232, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.8.ffn_up.weight", "offset": 376403456, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.8.ffn_norm.weight", "offset": 379399680, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 379403264, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 379403776, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.8.attn_output.weight", "offset": 379482624, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.8.attn_q.bias", "offset": 380034560, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 380038144, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.8.attn_v.bias", "offset": 380590080, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 380590592, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.9.attn_norm.weight", "offset": 380669440, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 380673024, "shape": [ 4864, 896 ], "size": 3584, "type": 6 }, { "name": "blk.9.ffn_gate.weight", "offset": 383669248, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.9.ffn_up.weight", "offset": 386665472, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.9.ffn_norm.weight", "offset": 389661696, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 389665280, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 389665792, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.9.attn_output.weight", "offset": 389744640, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.9.attn_q.bias", "offset": 390296576, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 390300160, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.9.attn_v.bias", "offset": 390852096, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 390852608, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "output_norm.weight", "offset": 390931456, "shape": [ 896 ], "size": 78848, "type": 0 } ], "version": 3 }