qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 9, "general.name": "Qwen2-1.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 12, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 1536, "qwen2.feed_forward_length": 8960, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 1543714304, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 191439360, "shape": [ 1536 ], "size": 191439360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 191445504, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.0.ffn_gate.weight", "offset": 201767424, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 212089344, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 222411264, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 222417408, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 222418432, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.0.attn_output.weight", "offset": 222713344, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.0.attn_q.bias", "offset": 224482816, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 224488960, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.0.attn_v.bias", "offset": 226258432, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 226259456, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 226554368, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 226560512, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.1.ffn_gate.weight", "offset": 236882432, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.1.ffn_up.weight", "offset": 247204352, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.1.ffn_norm.weight", "offset": 257526272, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 257532416, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 257533440, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.1.attn_output.weight", "offset": 257828352, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.1.attn_q.bias", "offset": 259597824, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 259603968, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.1.attn_v.bias", "offset": 261373440, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 261374464, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 261669376, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 261675520, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.10.ffn_gate.weight", "offset": 271997440, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.10.ffn_up.weight", "offset": 282319360, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 292641280, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 292647424, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 292648448, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.10.attn_output.weight", "offset": 292943360, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.10.attn_q.bias", "offset": 294712832, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 294718976, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.10.attn_v.bias", "offset": 296488448, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 296489472, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 296784384, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 296790528, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.17.ffn_gate.weight", "offset": 307112448, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.11.ffn_up.weight", "offset": 317434368, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.11.ffn_norm.weight", "offset": 327756288, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 327762432, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 327763456, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.11.attn_output.weight", "offset": 328058368, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.11.attn_q.bias", "offset": 329827840, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 329833984, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.11.attn_v.bias", "offset": 331603456, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 331604480, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 331899392, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 331905536, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.12.ffn_gate.weight", "offset": 342227456, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.12.ffn_up.weight", "offset": 352549376, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 362871296, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 362877440, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 362878464, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.12.attn_output.weight", "offset": 363173376, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.12.attn_q.bias", "offset": 364942848, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 364948992, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.12.attn_v.bias", "offset": 366718464, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 366719488, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 367014400, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 367020544, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.13.ffn_gate.weight", "offset": 377342464, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.13.ffn_up.weight", "offset": 387664384, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 397986304, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 397992448, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 397993472, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.13.attn_output.weight", "offset": 398288384, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.13.attn_q.bias", "offset": 400057856, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 400064000, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.13.attn_v.bias", "offset": 401833472, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 401834496, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 402129408, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 402135552, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.14.ffn_gate.weight", "offset": 412457472, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.14.ffn_up.weight", "offset": 422779392, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 433101312, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 433107456, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 433108480, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.14.attn_output.weight", "offset": 433403392, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.14.attn_q.bias", "offset": 435172864, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 435179008, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.14.attn_v.bias", "offset": 436948480, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 436949504, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 437244416, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 437250560, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.15.ffn_gate.weight", "offset": 447572480, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.15.ffn_up.weight", "offset": 457894400, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.15.ffn_norm.weight", "offset": 468216320, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 468222464, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 468223488, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.15.attn_output.weight", "offset": 468518400, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.15.attn_q.bias", "offset": 470287872, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 470294016, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.15.attn_v.bias", "offset": 472063488, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 472064512, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 472359424, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 472365568, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.16.ffn_gate.weight", "offset": 482687488, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 493009408, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 503331328, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 503337472, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 503338496, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.16.attn_output.weight", "offset": 503633408, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.16.attn_q.bias", "offset": 505402880, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 505409024, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.16.attn_v.bias", "offset": 507178496, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 507179520, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.17.attn_norm.weight", "offset": 507474432, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 507480576, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.17.ffn_gate.weight", "offset": 517802496, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.17.ffn_up.weight", "offset": 528124416, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.17.ffn_norm.weight", "offset": 538446336, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 538452480, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 538453504, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.17.attn_output.weight", "offset": 538748416, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.17.attn_q.bias", "offset": 540517888, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 540524032, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.17.attn_v.bias", "offset": 542293504, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 542294528, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 542589440, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 542595584, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.18.ffn_gate.weight", "offset": 552917504, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.18.ffn_up.weight", "offset": 563239424, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 573561344, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 573567488, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 573568512, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.18.attn_output.weight", "offset": 573863424, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.18.attn_q.bias", "offset": 575632896, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 575639040, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.18.attn_v.bias", "offset": 577408512, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 577409536, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.19.attn_norm.weight", "offset": 577704448, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 577710592, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.19.ffn_gate.weight", "offset": 588032512, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.19.ffn_up.weight", "offset": 598354432, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.19.ffn_norm.weight", "offset": 608676352, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 608682496, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 608683520, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.19.attn_output.weight", "offset": 608978432, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.19.attn_q.bias", "offset": 610747904, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 610754048, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.19.attn_v.bias", "offset": 612523520, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 612524544, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 612819456, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 612825600, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.2.ffn_gate.weight", "offset": 623147520, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.2.ffn_up.weight", "offset": 633469440, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.2.ffn_norm.weight", "offset": 643791360, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 643797504, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 643798528, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.2.attn_output.weight", "offset": 644093440, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.2.attn_q.bias", "offset": 645862912, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 645869056, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.2.attn_v.bias", "offset": 647638528, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 647639552, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 647934464, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 647940608, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.20.ffn_gate.weight", "offset": 658262528, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.20.ffn_up.weight", "offset": 668584448, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.20.ffn_norm.weight", "offset": 678906368, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 678912512, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 678913536, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.20.attn_output.weight", "offset": 679208448, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.20.attn_q.bias", "offset": 680977920, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 680984064, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.20.attn_v.bias", "offset": 682753536, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 682754560, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 683049472, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 683055616, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.21.ffn_gate.weight", "offset": 693377536, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.21.ffn_up.weight", "offset": 703699456, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.21.ffn_norm.weight", "offset": 714021376, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 714027520, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 714028544, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 714323456, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.21.attn_q.bias", "offset": 716092928, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 716099072, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.21.attn_v.bias", "offset": 717868544, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 717869568, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.22.attn_norm.weight", "offset": 718164480, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 718170624, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.22.ffn_gate.weight", "offset": 728492544, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.22.ffn_up.weight", "offset": 738814464, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.22.ffn_norm.weight", "offset": 749136384, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 749142528, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 749143552, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.22.attn_output.weight", "offset": 749438464, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.22.attn_q.bias", "offset": 751207936, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 751214080, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.22.attn_v.bias", "offset": 752983552, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 752984576, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 753279488, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 753285632, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.23.ffn_gate.weight", "offset": 763607552, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.23.ffn_up.weight", "offset": 773929472, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.23.ffn_norm.weight", "offset": 784251392, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 784257536, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 784258560, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.23.attn_output.weight", "offset": 784553472, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.23.attn_q.bias", "offset": 786322944, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 786329088, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.23.attn_v.bias", "offset": 788098560, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 788099584, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.24.attn_norm.weight", "offset": 788394496, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 788400640, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.24.ffn_gate.weight", "offset": 798722560, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.24.ffn_up.weight", "offset": 809044480, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.24.ffn_norm.weight", "offset": 819366400, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 819372544, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 819373568, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.24.attn_output.weight", "offset": 819668480, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.24.attn_q.bias", "offset": 821437952, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 821444096, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.24.attn_v.bias", "offset": 823213568, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 823214592, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.25.attn_norm.weight", "offset": 823509504, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 823515648, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.25.ffn_gate.weight", "offset": 833837568, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.25.ffn_up.weight", "offset": 844159488, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.25.ffn_norm.weight", "offset": 854481408, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 854487552, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 854488576, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.25.attn_output.weight", "offset": 854783488, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.25.attn_q.bias", "offset": 856552960, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 856559104, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.25.attn_v.bias", "offset": 858328576, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 858329600, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.26.attn_norm.weight", "offset": 858624512, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 858630656, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.26.ffn_gate.weight", "offset": 868952576, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.26.ffn_up.weight", "offset": 879274496, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.26.ffn_norm.weight", "offset": 889596416, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 889602560, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 889603584, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.26.attn_output.weight", "offset": 889898496, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.26.attn_q.bias", "offset": 891667968, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 891674112, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.26.attn_v.bias", "offset": 893443584, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 893444608, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.27.attn_norm.weight", "offset": 893739520, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 893745664, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.27.ffn_gate.weight", "offset": 904067584, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.27.ffn_up.weight", "offset": 914389504, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.27.ffn_norm.weight", "offset": 924711424, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 924717568, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 924718592, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.27.attn_output.weight", "offset": 925013504, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.27.attn_q.bias", "offset": 926782976, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 926789120, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.27.attn_v.bias", "offset": 928558592, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 928559616, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 928854528, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "\u0000\u0000\u0000\u0000*\ufffdC\u0000\u0000\u0000\u0000\u0015\u0000\u0000\u0000\u0000\u0000\u0000\u0000bl", "offset": 928860672, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.3.ffn_gate.weight", "offset": 939182592, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.3.ffn_up.weight", "offset": 949504512, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.3.ffn_norm.weight", "offset": 959826432, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 959832576, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 959833600, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.3.attn_output.weight", "offset": 960128512, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.3.attn_q.bias", "offset": 961897984, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 961904128, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.3.attn_v.bias", "offset": 963673600, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 963674624, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 963969536, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 963975680, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.4.ffn_gate.weight", "offset": 974297600, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.4.ffn_up.weight", "offset": 984619520, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.4.ffn_norm.weight", "offset": 994941440, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 994947584, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 994948608, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.4.attn_output.weight", "offset": 995243520, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.4.attn_q.bias", "offset": 997012992, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 997019136, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.4.attn_v.bias", "offset": 998788608, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 998789632, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 999084544, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 999090688, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.5.ffn_gate.weight", "offset": 1009412608, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.5.ffn_up.weight", "offset": 1019734528, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.5.ffn_norm.weight", "offset": 1030056448, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 1030062592, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1030063616, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.5.attn_output.weight", "offset": 1030358528, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.5.attn_q.bias", "offset": 1032128000, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1032134144, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.5.attn_v.bias", "offset": 1033903616, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 1033904640, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 1034199552, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1034205696, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.6.ffn_gate.weight", "offset": 1044527616, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.6.ffn_up.weight", "offset": 1054849536, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.6.ffn_norm.weight", "offset": 1065171456, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 1065177600, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1065178624, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 1065473536, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.6.attn_q.bias", "offset": 1067243008, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1067249152, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.6.attn_v.bias", "offset": 1069018624, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 1069019648, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.7.attn_norm.weight", "offset": 1069314560, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1069320704, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.7.ffn_gate.weight", "offset": 1079642624, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 1089964544, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.7.ffn_norm.weight", "offset": 1100286464, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 1100292608, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1100293632, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.7.attn_output.weight", "offset": 1100588544, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.7.attn_q.bias", "offset": 1102358016, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 1102364160, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.7.attn_v.bias", "offset": 1104133632, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 1104134656, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.8.attn_norm.weight", "offset": 1104429568, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1104435712, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.8.ffn_gate.weight", "offset": 1114757632, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.8.ffn_up.weight", "offset": 1125079552, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.8.ffn_norm.weight", "offset": 1135401472, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 1135407616, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1135408640, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.8.attn_output.weight", "offset": 1135703552, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.8.attn_q.bias", "offset": 1137473024, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 1137479168, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.8.attn_v.bias", "offset": 1139248640, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 1139249664, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 1139544576, "shape": [ 1536 ], "size": 294912, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1139550720, "shape": [ 8960, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.9.ffn_gate.weight", "offset": 1149872640, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.9.ffn_up.weight", "offset": 1160194560, "shape": [ 1536, 8960 ], "size": 10321920, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 1170516480, "shape": [ 1536 ], "size": 10321920, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 1170522624, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1170523648, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "blk.9.attn_output.weight", "offset": 1170818560, "shape": [ 1536, 1536 ], "size": 294912, "type": 7 }, { "name": "blk.9.attn_q.bias", "offset": 1172588032, "shape": [ 1536 ], "size": 1769472, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1172594176, "shape": [ 1536, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.9.attn_v.bias", "offset": 1174363648, "shape": [ 256 ], "size": 1769472, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1174364672, "shape": [ 1536, 256 ], "size": 1024, "type": 7 }, { "name": "output_norm.weight", "offset": 1174659584, "shape": [ 1536 ], "size": 294912, "type": 0 } ], "version": 3 }