qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 10, "general.name": "Qwen2-1.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 12, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 1536, "qwen2.feed_forward_length": 8960, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 1543714304, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 191439360, "shape": [ 1536 ], "size": 191439360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 191445504, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 197359104, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.0.ffn_up.weight", "offset": 201874944, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.0.ffn_norm.weight", "offset": 206390784, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 206396928, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 206397952, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.0.attn_output.weight", "offset": 206526976, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.0.attn_q.bias", "offset": 207540736, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 207546880, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.0.attn_v.bias", "offset": 208321024, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 208322048, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.1.attn_norm.weight", "offset": 208543232, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 208549376, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.1.ffn_gate.weight", "offset": 214462976, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.1.ffn_up.weight", "offset": 218978816, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.1.ffn_norm.weight", "offset": 223494656, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 223500800, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 223501824, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.1.attn_output.weight", "offset": 223630848, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.1.attn_q.bias", "offset": 224644608, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 224650752, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.1.attn_v.bias", "offset": 225424896, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 225425920, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 225647104, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 225653248, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.10.ffn_gate.weight", "offset": 231566848, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.10.ffn_up.weight", "offset": 236082688, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.10.ffn_norm.weight", "offset": 240598528, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 240604672, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 240605696, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.10.attn_output.weight", "offset": 240734720, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.10.attn_q.bias", "offset": 241748480, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 241754624, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.10.attn_v.bias", "offset": 242528768, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 242529792, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 242750976, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 242757120, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.11.ffn_gate.weight", "offset": 248670720, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.11.ffn_up.weight", "offset": 253186560, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.11.ffn_norm.weight", "offset": 257702400, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 257708544, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 257709568, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.11.attn_output.weight", "offset": 257838592, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.11.attn_q.bias", "offset": 258852352, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 258858496, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.11.attn_v.bias", "offset": 259632640, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 259633664, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 259854848, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 259860992, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 265774592, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.12.ffn_up.weight", "offset": 270290432, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.12.ffn_norm.weight", "offset": 274806272, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 274812416, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 274813440, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.12.attn_output.weight", "offset": 274942464, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.12.attn_q.bias", "offset": 275956224, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 275962368, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.12.attn_v.bias", "offset": 276736512, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 276737536, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 276958720, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 276964864, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 282878464, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.13.ffn_up.weight", "offset": 287394304, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.13.ffn_norm.weight", "offset": 291910144, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 291916288, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 291917312, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.13.attn_output.weight", "offset": 292046336, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.13.attn_q.bias", "offset": 293060096, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 293066240, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.13.attn_v.bias", "offset": 293840384, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 293841408, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 294062592, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 294068736, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.14.ffn_gate.weight", "offset": 299982336, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.14.ffn_up.weight", "offset": 304498176, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.14.ffn_norm.weight", "offset": 309014016, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 309020160, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 309021184, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.14.attn_output.weight", "offset": 309150208, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.14.attn_q.bias", "offset": 310163968, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 310170112, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.14.attn_v.bias", "offset": 310944256, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 310945280, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 311166464, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 311172608, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.15.ffn_gate.weight", "offset": 317086208, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.15.ffn_up.weight", "offset": 321602048, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.15.ffn_norm.weight", "offset": 326117888, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 326124032, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 326125056, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.15.attn_output.weight", "offset": 326254080, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.15.attn_q.bias", "offset": 327267840, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 327273984, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.15.attn_v.bias", "offset": 328048128, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 328049152, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 328270336, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 328276480, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 334190080, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.16.ffn_up.weight", "offset": 338705920, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.16.ffn_norm.weight", "offset": 343221760, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 343227904, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 343228928, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.16.attn_output.weight", "offset": 343357952, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.16.attn_q.bias", "offset": 344371712, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 344377856, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.16.attn_v.bias", "offset": 345152000, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 345153024, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 345374208, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 345380352, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 351293952, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.17.ffn_up.weight", "offset": 355809792, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.17.ffn_norm.weight", "offset": 360325632, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 360331776, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 360332800, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.17.attn_output.weight", "offset": 360461824, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.17.attn_q.bias", "offset": 361475584, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 361481728, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.17.attn_v.bias", "offset": 362255872, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 362256896, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 362478080, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 362484224, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.18.ffn_gate.weight", "offset": 368397824, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.18.ffn_up.weight", "offset": 372913664, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.18.ffn_norm.weight", "offset": 377429504, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 377435648, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 377436672, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.18.attn_output.weight", "offset": 377565696, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.18.attn_q.bias", "offset": 378579456, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 378585600, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.18.attn_v.bias", "offset": 379359744, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 379360768, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 379581952, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 379588096, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.19.ffn_gate.weight", "offset": 385501696, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.19.ffn_up.weight", "offset": 390017536, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.19.ffn_norm.weight", "offset": 394533376, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 394539520, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 394540544, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.19.attn_output.weight", "offset": 394669568, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.19.attn_q.bias", "offset": 395683328, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 395689472, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.19.attn_v.bias", "offset": 396463616, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 396464640, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 396685824, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 396691968, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.2.ffn_gate.weight", "offset": 402605568, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.2.ffn_up.weight", "offset": 407121408, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.2.ffn_norm.weight", "offset": 411637248, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 411643392, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 411644416, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.2.attn_output.weight", "offset": 411773440, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.2.attn_q.bias", "offset": 412787200, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 412793344, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.2.attn_v.bias", "offset": 413567488, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 413568512, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 413789696, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 413795840, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.20.ffn_gate.weight", "offset": 419709440, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.20.ffn_up.weight", "offset": 424225280, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.20.ffn_norm.weight", "offset": 428741120, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 428747264, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 428748288, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.20.attn_output.weight", "offset": 428877312, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.20.attn_q.bias", "offset": 429891072, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 429897216, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.20.attn_v.bias", "offset": 430671360, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 430672384, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 430893568, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 430899712, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.21.ffn_gate.weight", "offset": 436813312, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.21.ffn_up.weight", "offset": 441329152, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.21.ffn_norm.weight", "offset": 445844992, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 445851136, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 445852160, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.21.attn_output.weight", "offset": 445981184, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.21.attn_q.bias", "offset": 446994944, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 447001088, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.21.attn_v.bias", "offset": 447775232, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 447776256, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 447997440, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 448003584, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.22.ffn_gate.weight", "offset": 453917184, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.22.ffn_up.weight", "offset": 458433024, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.22.ffn_norm.weight", "offset": 462948864, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 462955008, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 462956032, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.22.attn_output.weight", "offset": 463085056, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.22.attn_q.bias", "offset": 464098816, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 464104960, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.22.attn_v.bias", "offset": 464879104, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 464880128, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 465101312, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 465107456, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.23.ffn_gate.weight", "offset": 471021056, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.23.ffn_up.weight", "offset": 475536896, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.23.ffn_norm.weight", "offset": 480052736, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 480058880, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 480059904, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.23.attn_output.weight", "offset": 480188928, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.23.attn_q.bias", "offset": 481202688, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 481208832, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.23.attn_v.bias", "offset": 481982976, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 481984000, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 482205184, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 482211328, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.24.ffn_gate.weight", "offset": 488124928, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.24.ffn_up.weight", "offset": 492640768, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.24.ffn_norm.weight", "offset": 497156608, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 497162752, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 497163776, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.24.attn_output.weight", "offset": 497292800, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.24.attn_q.bias", "offset": 498306560, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 498312704, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.24.attn_v.bias", "offset": 499086848, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 499087872, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 499309056, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 499315200, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.25.ffn_gate.weight", "offset": 505228800, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.25.ffn_up.weight", "offset": 509744640, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.25.ffn_norm.weight", "offset": 514260480, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 514266624, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 514267648, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.25.attn_output.weight", "offset": 514396672, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.25.attn_q.bias", "offset": 515410432, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 515416576, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.25.attn_v.bias", "offset": 516190720, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 516191744, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 516412928, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 516419072, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.26.ffn_gate.weight", "offset": 522332672, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.26.ffn_up.weight", "offset": 526848512, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.26.ffn_norm.weight", "offset": 531364352, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 531370496, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 531371520, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.26.attn_output.weight", "offset": 531500544, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.26.attn_q.bias", "offset": 532514304, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 532520448, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.26.attn_v.bias", "offset": 533294592, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 533295616, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 533516800, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 533522944, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.27.ffn_gate.weight", "offset": 539436544, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.27.ffn_up.weight", "offset": 543952384, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.27.ffn_norm.weight", "offset": 548468224, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 548474368, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 548475392, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.27.attn_output.weight", "offset": 548604416, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.27.attn_q.bias", "offset": 549618176, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 549624320, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.27.attn_v.bias", "offset": 550398464, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 550399488, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 550620672, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 550626816, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.3.ffn_gate.weight", "offset": 556540416, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.3.ffn_up.weight", "offset": 561056256, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.3.ffn_norm.weight", "offset": 565572096, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 565578240, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 565579264, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.3.attn_output.weight", "offset": 565708288, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.3.attn_q.bias", "offset": 566722048, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 566728192, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.3.attn_v.bias", "offset": 567502336, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 567503360, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 567724544, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 567730688, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.4.ffn_gate.weight", "offset": 573644288, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.4.ffn_up.weight", "offset": 578160128, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.4.ffn_norm.weight", "offset": 582675968, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 582682112, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 582683136, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.4.attn_output.weight", "offset": 582812160, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.4.attn_q.bias", "offset": 583825920, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 583832064, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.4.attn_v.bias", "offset": 584606208, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 584607232, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 584828416, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 584834560, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.5.ffn_gate.weight", "offset": 590748160, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.5.ffn_up.weight", "offset": 595264000, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.5.ffn_norm.weight", "offset": 599779840, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 599785984, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 599787008, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.5.attn_output.weight", "offset": 599916032, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.5.attn_q.bias", "offset": 600929792, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 600935936, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.5.attn_v.bias", "offset": 601710080, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 601711104, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 601932288, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 601938432, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.6.ffn_gate.weight", "offset": 607852032, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.6.ffn_up.weight", "offset": 612367872, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.6.ffn_norm.weight", "offset": 616883712, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 616889856, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 616890880, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.6.attn_output.weight", "offset": 617019904, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.6.attn_q.bias", "offset": 618033664, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 618039808, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.6.attn_v.bias", "offset": 618813952, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 618814976, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 619036160, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 619042304, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.7.ffn_gate.weight", "offset": 624955904, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.7.ffn_up.weight", "offset": 629471744, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.7.ffn_norm.weight", "offset": 633987584, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 633993728, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 633994752, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.7.attn_output.weight", "offset": 634123776, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.7.attn_q.bias", "offset": 635137536, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 635143680, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.7.attn_v.bias", "offset": 635917824, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 635918848, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 636140032, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 636146176, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.8.ffn_gate.weight", "offset": 642059776, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.8.ffn_up.weight", "offset": 646575616, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.8.ffn_norm.weight", "offset": 651091456, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 651097600, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 651098624, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.8.attn_output.weight", "offset": 651227648, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.8.attn_q.bias", "offset": 652241408, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 652247552, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.8.attn_v.bias", "offset": 653021696, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 653022720, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 653243904, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 653250048, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.9.ffn_gate.weight", "offset": 659163648, "shape": [ 1536, 8960 ], "size": 5913600, "type": 10 }, { "name": "blk.9.ffn_up.weight", "offset": 663679488, "shape": [ 1536, 8960 ], "size": 4515840, "type": 10 }, { "name": "blk.9.ffn_norm.weight", "offset": 668195328, "shape": [ 1536 ], "size": 4515840, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 668201472, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 668202496, "shape": [ 1536, 256 ], "size": 1024, "type": 10 }, { "name": "blk.9.attn_output.weight", "offset": 668331520, "shape": [ 1536, 1536 ], "size": 129024, "type": 11 }, { "name": "blk.9.attn_q.bias", "offset": 669345280, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 669351424, "shape": [ 1536, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.9.attn_v.bias", "offset": 670125568, "shape": [ 256 ], "size": 774144, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 670126592, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "output_norm.weight", "offset": 670347776, "shape": [ 1536 ], "size": 221184, "type": 0 } ], "version": 3 }