qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 7, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.0.ffn_gate.weight", "offset": 149277184, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 153907712, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 158538240, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 158541824, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 158542336, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 158664192, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.0.attn_q.bias", "offset": 159517184, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 159520768, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.0.attn_v.bias", "offset": 160373760, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 160374272, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 160496128, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 160499712, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.1.ffn_gate.weight", "offset": 165130240, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.1.ffn_up.weight", "offset": 169760768, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.1.ffn_norm.weight", "offset": 174391296, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 174394880, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 174395392, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 174517248, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.1.attn_q.bias", "offset": 175370240, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 175373824, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.1.attn_v.bias", "offset": 176226816, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 176227328, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 176349184, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 176352768, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.10.ffn_gate.weight", "offset": 180983296, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.10.ffn_up.weight", "offset": 185613824, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.10.ffn_norm.weight", "offset": 190244352, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 190247936, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 190248448, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 190370304, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.10.attn_q.bias", "offset": 191223296, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 191226880, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.10.attn_v.bias", "offset": 192079872, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 192080384, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 192202240, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 192205824, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "\u0000\u0000\u0000\u0000\ufffd\u0003\u0000\u0000\u0000\u0000\u0000\u0000\b\u0000\u0000\u0000\u00008 \u0011\u0000\u0000", "offset": 196836352, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.11.ffn_up.weight", "offset": 201466880, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.11.ffn_norm.weight", "offset": 206097408, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 206100992, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 206101504, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 206223360, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.11.attn_q.bias", "offset": 207076352, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 207079936, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.11.attn_v.bias", "offset": 207932928, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 207933440, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.12.attn_norm.weight", "offset": 208055296, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 208058880, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.12.ffn_gate.weight", "offset": 212689408, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.12.ffn_up.weight", "offset": 217319936, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.12.ffn_norm.weight", "offset": 221950464, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 221954048, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 221954560, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 222076416, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.12.attn_q.bias", "offset": 222929408, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 222932992, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.12.attn_v.bias", "offset": 223785984, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 223786496, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.13.attn_norm.weight", "offset": 223908352, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 223911936, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.13.ffn_gate.weight", "offset": 228542464, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.13.ffn_up.weight", "offset": 233172992, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.13.ffn_norm.weight", "offset": 237803520, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 237807104, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 237807616, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 237929472, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.13.attn_q.bias", "offset": 238782464, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 238786048, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.13.attn_v.bias", "offset": 239639040, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 239639552, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 239761408, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 239764992, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.14.ffn_gate.weight", "offset": 244395520, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.14.ffn_up.weight", "offset": 249026048, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.14.ffn_norm.weight", "offset": 253656576, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 253660160, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 253660672, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 253782528, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.14.attn_q.bias", "offset": 254635520, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 254639104, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.14.attn_v.bias", "offset": 255492096, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 255492608, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.15.attn_norm.weight", "offset": 255614464, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 255618048, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.15.ffn_gate.weight", "offset": 260248576, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.15.ffn_up.weight", "offset": 264879104, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.15.ffn_norm.weight", "offset": 269509632, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 269513216, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 269513728, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 269635584, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.15.attn_q.bias", "offset": 270488576, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 270492160, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.15.attn_v.bias", "offset": 271345152, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 271345664, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.16.attn_norm.weight", "offset": 271467520, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 271471104, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.16.ffn_gate.weight", "offset": 276101632, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.16.ffn_up.weight", "offset": 280732160, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.16.ffn_norm.weight", "offset": 285362688, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 285366272, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 285366784, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 285488640, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.16.attn_q.bias", "offset": 286341632, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 286345216, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.16.attn_v.bias", "offset": 287198208, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 287198720, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 287320576, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 287324160, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.17.ffn_gate.weight", "offset": 291954688, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.17.ffn_up.weight", "offset": 296585216, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.17.ffn_norm.weight", "offset": 301215744, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 301219328, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 301219840, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 301341696, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.17.attn_q.bias", "offset": 302194688, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 302198272, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.17.attn_v.bias", "offset": 303051264, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 303051776, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.18.attn_norm.weight", "offset": 303173632, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 303177216, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.18.ffn_gate.weight", "offset": 307807744, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.18.ffn_up.weight", "offset": 312438272, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 317068800, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 317072384, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 317072896, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 317194752, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.18.attn_q.bias", "offset": 318047744, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 318051328, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.18.attn_v.bias", "offset": 318904320, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 318904832, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.19.attn_norm.weight", "offset": 319026688, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 319030272, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.19.ffn_gate.weight", "offset": 323660800, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.19.ffn_up.weight", "offset": 328291328, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.19.ffn_norm.weight", "offset": 332921856, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 332925440, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 332925952, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 333047808, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.19.attn_q.bias", "offset": 333900800, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 333904384, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.19.attn_v.bias", "offset": 334757376, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 334757888, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 334879744, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 334883328, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.2.ffn_gate.weight", "offset": 339513856, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.2.ffn_up.weight", "offset": 344144384, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.2.ffn_norm.weight", "offset": 348774912, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 348778496, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 348779008, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 348900864, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.2.attn_q.bias", "offset": 349753856, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 349757440, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.2.attn_v.bias", "offset": 350610432, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 350610944, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.20.attn_norm.weight", "offset": 350732800, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 350736384, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.20.ffn_gate.weight", "offset": 355366912, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.20.ffn_up.weight", "offset": 359997440, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.20.ffn_norm.weight", "offset": 364627968, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 364631552, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 364632064, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 364753920, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.20.attn_q.bias", "offset": 365606912, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 365610496, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.20.attn_v.bias", "offset": 366463488, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 366464000, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.21.attn_norm.weight", "offset": 366585856, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 366589440, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.21.ffn_gate.weight", "offset": 371219968, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.21.ffn_up.weight", "offset": 375850496, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.21.ffn_norm.weight", "offset": 380481024, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 380484608, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 380485120, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 380606976, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.21.attn_q.bias", "offset": 381459968, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 381463552, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.21.attn_v.bias", "offset": 382316544, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 382317056, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 382438912, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 382442496, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.22.ffn_gate.weight", "offset": 387073024, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.22.ffn_up.weight", "offset": 391703552, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.22.ffn_norm.weight", "offset": 396334080, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 396337664, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 396338176, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 396460032, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.22.attn_q.bias", "offset": 397313024, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 397316608, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.22.attn_v.bias", "offset": 398169600, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 398170112, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.23.attn_norm.weight", "offset": 398291968, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 398295552, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.23.ffn_gate.weight", "offset": 402926080, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.23.ffn_up.weight", "offset": 407556608, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.23.ffn_norm.weight", "offset": 412187136, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 412190720, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 412191232, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 412313088, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.23.attn_q.bias", "offset": 413166080, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 413169664, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.23.attn_v.bias", "offset": 414022656, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 414023168, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.3.attn_norm.weight", "offset": 414145024, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 414148608, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.3.ffn_gate.weight", "offset": 418779136, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.3.ffn_up.weight", "offset": 423409664, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.3.ffn_norm.weight", "offset": 428040192, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 428043776, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 428044288, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 428166144, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.3.attn_q.bias", "offset": 429019136, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 429022720, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.3.attn_v.bias", "offset": 429875712, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 429876224, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 429998080, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 430001664, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.4.ffn_gate.weight", "offset": 434632192, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.4.ffn_up.weight", "offset": 439262720, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.4.ffn_norm.weight", "offset": 443893248, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 443896832, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 443897344, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 444019200, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.4.attn_q.bias", "offset": 444872192, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 444875776, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.4.attn_v.bias", "offset": 445728768, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 445729280, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.5.attn_norm.weight", "offset": 445851136, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 445854720, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.5.ffn_gate.weight", "offset": 450485248, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.5.ffn_up.weight", "offset": 455115776, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.5.ffn_norm.weight", "offset": 459746304, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 459749888, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 459750400, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 459872256, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.5.attn_q.bias", "offset": 460725248, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 460728832, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.5.attn_v.bias", "offset": 461581824, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 461582336, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.6.attn_norm.weight", "offset": 461704192, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 461707776, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.6.ffn_gate.weight", "offset": 466338304, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.6.ffn_up.weight", "offset": 470968832, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.6.ffn_norm.weight", "offset": 475599360, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 475602944, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 475603456, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 475725312, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.6.attn_q.bias", "offset": 476578304, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 476581888, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.6.attn_v.bias", "offset": 477434880, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 477435392, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.7.attn_norm.weight", "offset": 477557248, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 477560832, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.7.ffn_gate.weight", "offset": 482191360, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 486821888, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.7.ffn_norm.weight", "offset": 491452416, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 491456000, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 491456512, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 491578368, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.7.attn_q.bias", "offset": 492431360, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 492434944, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.7.attn_v.bias", "offset": 493287936, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 493288448, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.8.attn_norm.weight", "offset": 493410304, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 493413888, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.8.ffn_gate.weight", "offset": 498044416, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.8.ffn_up.weight", "offset": 502674944, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.8.ffn_norm.weight", "offset": 507305472, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 507309056, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 507309568, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 507431424, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.8.attn_q.bias", "offset": 508284416, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 508288000, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.8.attn_v.bias", "offset": 509140992, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 509141504, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 509263360, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 509266944, "shape": [ 4864, 896 ], "size": 3584, "type": 8 }, { "name": "blk.9.ffn_gate.weight", "offset": 513897472, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.9.ffn_up.weight", "offset": 518528000, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.9.ffn_norm.weight", "offset": 523158528, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 523162112, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 523162624, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 523284480, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.9.attn_q.bias", "offset": 524137472, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 524141056, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.9.attn_v.bias", "offset": 524994048, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 524994560, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "output_norm.weight", "offset": 525116416, "shape": [ 896 ], "size": 121856, "type": 0 } ], "version": 3 }