qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 3, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.0.ffn_gate.weight", "offset": 147370496, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.0.ffn_up.weight", "offset": 150094336, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.0.ffn_norm.weight", "offset": 152818176, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 152821760, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 152822272, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.0.attn_output.weight", "offset": 152893952, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.0.attn_q.bias", "offset": 153395712, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 153399296, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.0.attn_v.bias", "offset": 153901056, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 153901568, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.1.attn_norm.weight", "offset": 153973248, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 153976832, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.1.ffn_gate.weight", "offset": 156700672, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.1.ffn_up.weight", "offset": 159424512, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.1.ffn_norm.weight", "offset": 162148352, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 162151936, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 162152448, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.1.attn_output.weight", "offset": 162224128, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.1.attn_q.bias", "offset": 162725888, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 162729472, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.1.attn_v.bias", "offset": 163231232, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 163231744, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.10.attn_norm.weight", "offset": 163303424, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 163307008, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.10.ffn_gate.weight", "offset": 166030848, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.10.ffn_up.weight", "offset": 168754688, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.10.ffn_norm.weight", "offset": 171478528, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 171482112, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 171482624, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.10.attn_output.weight", "offset": 171554304, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.10.attn_q.bias", "offset": 172056064, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 172059648, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.10.attn_v.bias", "offset": 172561408, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 172561920, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.11.attn_norm.weight", "offset": 172633600, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 172637184, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.11.ffn_gate.weight", "offset": 175361024, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.11.ffn_up.weight", "offset": 178084864, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.11.ffn_norm.weight", "offset": 180808704, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 180812288, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 180812800, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.11.attn_output.weight", "offset": 180884480, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.11.attn_q.bias", "offset": 181386240, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 181389824, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.11.attn_v.bias", "offset": 181891584, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 181892096, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.12.attn_norm.weight", "offset": 181963776, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 181967360, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.12.ffn_gate.weight", "offset": 184691200, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.12.ffn_up.weight", "offset": 187415040, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.12.ffn_norm.weight", "offset": 190138880, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 190142464, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 190142976, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.12.attn_output.weight", "offset": 190214656, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.12.attn_q.bias", "offset": 190716416, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 190720000, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.12.attn_v.bias", "offset": 191221760, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 191222272, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.13.attn_norm.weight", "offset": 191293952, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 191297536, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.13.ffn_gate.weight", "offset": 194021376, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.13.ffn_up.weight", "offset": 196745216, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.13.ffn_norm.weight", "offset": 199469056, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 199472640, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 199473152, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.13.attn_output.weight", "offset": 199544832, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.13.attn_q.bias", "offset": 200046592, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 200050176, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.13.attn_v.bias", "offset": 200551936, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 200552448, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.14.attn_norm.weight", "offset": 200624128, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 200627712, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.14.ffn_gate.weight", "offset": 203351552, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.14.ffn_up.weight", "offset": 206075392, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.14.ffn_norm.weight", "offset": 208799232, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 208802816, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 208803328, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.14.attn_output.weight", "offset": 208875008, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.14.attn_q.bias", "offset": 209376768, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 209380352, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.14.attn_v.bias", "offset": 209882112, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 209882624, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.15.attn_norm.weight", "offset": 209954304, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 209957888, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.15.ffn_gate.weight", "offset": 212681728, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.15.ffn_up.weight", "offset": 215405568, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.15.ffn_norm.weight", "offset": 218129408, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 218132992, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 218133504, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.15.attn_output.weight", "offset": 218205184, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.15.attn_q.bias", "offset": 218706944, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 218710528, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.15.attn_v.bias", "offset": 219212288, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 219212800, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.16.attn_norm.weight", "offset": 219284480, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 219288064, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.16.ffn_gate.weight", "offset": 222011904, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.16.ffn_up.weight", "offset": 224735744, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.16.ffn_norm.weight", "offset": 227459584, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 227463168, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 227463680, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.16.attn_output.weight", "offset": 227535360, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.16.attn_q.bias", "offset": 228037120, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 228040704, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.16.attn_v.bias", "offset": 228542464, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 228542976, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.17.attn_norm.weight", "offset": 228614656, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 228618240, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.17.ffn_gate.weight", "offset": 231342080, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.17.ffn_up.weight", "offset": 234065920, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.17.ffn_norm.weight", "offset": 236789760, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 236793344, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 236793856, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.17.attn_output.weight", "offset": 236865536, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.17.attn_q.bias", "offset": 237367296, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 237370880, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.17.attn_v.bias", "offset": 237872640, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 237873152, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.18.attn_norm.weight", "offset": 237944832, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 237948416, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.18.ffn_gate.weight", "offset": 240672256, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.18.ffn_up.weight", "offset": 243396096, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.18.ffn_norm.weight", "offset": 246119936, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 246123520, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 246124032, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.18.attn_output.weight", "offset": 246195712, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.18.attn_q.bias", "offset": 246697472, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 246701056, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.18.attn_v.bias", "offset": 247202816, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 247203328, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.19.attn_norm.weight", "offset": 247275008, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 247278592, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.19.ffn_gate.weight", "offset": 250002432, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.19.ffn_up.weight", "offset": 252726272, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.19.ffn_norm.weight", "offset": 255450112, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 255453696, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 255454208, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.19.attn_output.weight", "offset": 255525888, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.19.attn_q.bias", "offset": 256027648, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 256031232, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.19.attn_v.bias", "offset": 256532992, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 256533504, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.2.attn_norm.weight", "offset": 256605184, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 256608768, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.2.ffn_gate.weight", "offset": 259332608, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.2.ffn_up.weight", "offset": 262056448, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.2.ffn_norm.weight", "offset": 264780288, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 264783872, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 264784384, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.2.attn_output.weight", "offset": 264856064, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.2.attn_q.bias", "offset": 265357824, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 265361408, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.2.attn_v.bias", "offset": 265863168, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 265863680, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.20.attn_norm.weight", "offset": 265935360, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 265938944, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.20.ffn_gate.weight", "offset": 268662784, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.20.ffn_up.weight", "offset": 271386624, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.20.ffn_norm.weight", "offset": 274110464, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 274114048, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 274114560, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.20.attn_output.weight", "offset": 274186240, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.20.attn_q.bias", "offset": 274688000, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 274691584, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.20.attn_v.bias", "offset": 275193344, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 275193856, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.21.attn_norm.weight", "offset": 275265536, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 275269120, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.21.ffn_gate.weight", "offset": 277992960, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.21.ffn_up.weight", "offset": 280716800, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.21.ffn_norm.weight", "offset": 283440640, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 283444224, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 283444736, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.21.attn_output.weight", "offset": 283516416, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.21.attn_q.bias", "offset": 284018176, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 284021760, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.21.attn_v.bias", "offset": 284523520, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 284524032, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.22.attn_norm.weight", "offset": 284595712, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 284599296, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.22.ffn_gate.weight", "offset": 287323136, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.22.ffn_up.weight", "offset": 290046976, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.22.ffn_norm.weight", "offset": 292770816, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 292774400, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 292774912, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.22.attn_output.weight", "offset": 292846592, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.22.attn_q.bias", "offset": 293348352, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 293351936, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.22.attn_v.bias", "offset": 293853696, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 293854208, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.23.attn_norm.weight", "offset": 293925888, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 293929472, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.23.ffn_gate.weight", "offset": 296653312, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.23.ffn_up.weight", "offset": 299377152, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.23.ffn_norm.weight", "offset": 302100992, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 302104576, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 302105088, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.23.attn_output.weight", "offset": 302176768, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.23.attn_q.bias", "offset": 302678528, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 302682112, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.23.attn_v.bias", "offset": 303183872, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 303184384, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.3.attn_norm.weight", "offset": 303256064, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 303259648, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.3.ffn_gate.weight", "offset": 305983488, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.3.ffn_up.weight", "offset": 308707328, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.3.ffn_norm.weight", "offset": 311431168, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 311434752, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 311435264, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.3.attn_output.weight", "offset": 311506944, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.3.attn_q.bias", "offset": 312008704, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 312012288, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.3.attn_v.bias", "offset": 312514048, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 312514560, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.4.attn_norm.weight", "offset": 312586240, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 312589824, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.4.ffn_gate.weight", "offset": 315313664, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.4.ffn_up.weight", "offset": 318037504, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.4.ffn_norm.weight", "offset": 320761344, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 320764928, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 320765440, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.4.attn_output.weight", "offset": 320837120, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.4.attn_q.bias", "offset": 321338880, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 321342464, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.4.attn_v.bias", "offset": 321844224, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 321844736, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.5.attn_norm.weight", "offset": 321916416, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 321920000, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.5.ffn_gate.weight", "offset": 324643840, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.5.ffn_up.weight", "offset": 327367680, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.5.ffn_norm.weight", "offset": 330091520, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 330095104, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 330095616, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.5.attn_output.weight", "offset": 330167296, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.5.attn_q.bias", "offset": 330669056, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 330672640, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.5.attn_v.bias", "offset": 331174400, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 331174912, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.6.attn_norm.weight", "offset": 331246592, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 331250176, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.6.ffn_gate.weight", "offset": 333974016, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.6.ffn_up.weight", "offset": 336697856, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.6.ffn_norm.weight", "offset": 339421696, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 339425280, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 339425792, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.6.attn_output.weight", "offset": 339497472, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.6.attn_q.bias", "offset": 339999232, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 340002816, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.6.attn_v.bias", "offset": 340504576, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 340505088, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.7.attn_norm.weight", "offset": 340576768, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 340580352, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.7.ffn_gate.weight", "offset": 343304192, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.7.ffn_up.weight", "offset": 346028032, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.7.ffn_norm.weight", "offset": 348751872, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 348755456, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 348755968, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.7.attn_output.weight", "offset": 348827648, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.7.attn_q.bias", "offset": 349329408, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 349332992, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.7.attn_v.bias", "offset": 349834752, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 349835264, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.8.attn_norm.weight", "offset": 349906944, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 349910528, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.8.ffn_gate.weight", "offset": 352634368, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.8.ffn_up.weight", "offset": 355358208, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.8.ffn_norm.weight", "offset": 358082048, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 358085632, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 358086144, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.8.attn_output.weight", "offset": 358157824, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.8.attn_q.bias", "offset": 358659584, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 358663168, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.8.attn_v.bias", "offset": 359164928, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 359165440, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.9.attn_norm.weight", "offset": 359237120, "shape": [ 896 ], "size": 71680, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 359240704, "shape": [ 4864, 896 ], "size": 3584, "type": 3 }, { "name": "blk.9.ffn_gate.weight", "offset": 361964544, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.9.ffn_up.weight", "offset": 364688384, "shape": [ 896, 4864 ], "size": 2723840, "type": 3 }, { "name": "blk.9.ffn_norm.weight", "offset": 367412224, "shape": [ 896 ], "size": 2723840, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 367415808, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 367416320, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "blk.9.attn_output.weight", "offset": 367488000, "shape": [ 896, 896 ], "size": 71680, "type": 3 }, { "name": "blk.9.attn_q.bias", "offset": 367989760, "shape": [ 896 ], "size": 501760, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 367993344, "shape": [ 896, 896 ], "size": 3584, "type": 3 }, { "name": "blk.9.attn_v.bias", "offset": 368495104, "shape": [ 128 ], "size": 501760, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 368495616, "shape": [ 896, 128 ], "size": 512, "type": 3 }, { "name": "output_norm.weight", "offset": 368567296, "shape": [ 896 ], "size": 71680, "type": 0 } ], "version": 3 }