qwen2

Qwen2 is a new series of large language models from Alibaba group

0.5B 1.5B 7B 72B

273.2K Pulls Updated 4 weeks ago

97 Tags

... /
model
2a79e0e04078 · 3.1GB
{ "metadata": { "general.architecture": "qwen2", "general.file_type": 1, "general.name": "Qwen2-1.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 12, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 1536, "qwen2.feed_forward_length": 8960, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 1543714304, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 466747392, "shape": [ 1536 ], "size": 466747392, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 466753536, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.0.ffn_gate.weight", "offset": 494278656, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.0.ffn_up.weight", "offset": 521803776, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.0.ffn_norm.weight", "offset": 549328896, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 549335040, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 549336064, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.0.attn_output.weight", "offset": 550122496, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.0.attn_q.bias", "offset": 554841088, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 554847232, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.0.attn_v.bias", "offset": 559565824, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 559566848, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.1.attn_norm.weight", "offset": 560353280, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 560359424, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.1.ffn_gate.weight", "offset": 587884544, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.1.ffn_up.weight", "offset": 615409664, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.1.ffn_norm.weight", "offset": 642934784, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 642940928, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 642941952, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.1.attn_output.weight", "offset": 643728384, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.1.attn_q.bias", "offset": 648446976, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 648453120, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.1.attn_v.bias", "offset": 653171712, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 653172736, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.10.attn_norm.weight", "offset": 653959168, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 653965312, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.10.ffn_gate.weight", "offset": 681490432, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.10.ffn_up.weight", "offset": 709015552, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.10.ffn_norm.weight", "offset": 736540672, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 736546816, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 736547840, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.10.attn_output.weight", "offset": 737334272, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.10.attn_q.bias", "offset": 742052864, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 742059008, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.10.attn_v.bias", "offset": 746777600, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 746778624, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.11.attn_norm.weight", "offset": 747565056, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 747571200, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.11.ffn_gate.weight", "offset": 775096320, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.11.ffn_up.weight", "offset": 802621440, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.11.ffn_norm.weight", "offset": 830146560, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 830152704, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 830153728, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.11.attn_output.weight", "offset": 830940160, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.11.attn_q.bias", "offset": 835658752, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 835664896, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.11.attn_v.bias", "offset": 840383488, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 840384512, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 841170944, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 841177088, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.12.ffn_gate.weight", "offset": 868702208, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.12.ffn_up.weight", "offset": 896227328, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.12.ffn_norm.weight", "offset": 923752448, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 923758592, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 923759616, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.12.attn_output.weight", "offset": 924546048, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.12.attn_q.bias", "offset": 929264640, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 929270784, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.12.attn_v.bias", "offset": 933989376, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 933990400, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.13.attn_norm.weight", "offset": 934776832, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 934782976, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.13.ffn_gate.weight", "offset": 962308096, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.13.ffn_up.weight", "offset": 989833216, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.13.ffn_norm.weight", "offset": 1017358336, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 1017364480, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1017365504, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.13.attn_output.weight", "offset": 1018151936, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.13.attn_q.bias", "offset": 1022870528, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1022876672, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.13.attn_v.bias", "offset": 1027595264, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 1027596288, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 1028382720, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1028388864, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.14.ffn_gate.weight", "offset": 1055913984, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.14.ffn_up.weight", "offset": 1083439104, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.14.ffn_norm.weight", "offset": 1110964224, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 1110970368, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1110971392, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.14.attn_output.weight", "offset": 1111757824, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.14.attn_q.bias", "offset": 1116476416, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 1116482560, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.14.attn_v.bias", "offset": 1121201152, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 1121202176, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.15.attn_norm.weight", "offset": 1121988608, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1121994752, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.15.ffn_gate.weight", "offset": 1149519872, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.15.ffn_up.weight", "offset": 1177044992, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.15.ffn_norm.weight", "offset": 1204570112, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 1204576256, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1204577280, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.15.attn_output.weight", "offset": 1205363712, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.15.attn_q.bias", "offset": 1210082304, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 1210088448, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.15.attn_v.bias", "offset": 1214807040, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 1214808064, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.16.attn_norm.weight", "offset": 1215594496, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 1215600640, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.16.ffn_gate.weight", "offset": 1243125760, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.16.ffn_up.weight", "offset": 1270650880, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.16.ffn_norm.weight", "offset": 1298176000, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 1298182144, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1298183168, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.16.attn_output.weight", "offset": 1298969600, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.16.attn_q.bias", "offset": 1303688192, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1303694336, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.16.attn_v.bias", "offset": 1308412928, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 1308413952, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 1309200384, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 1309206528, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.17.ffn_gate.weight", "offset": 1336731648, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.17.ffn_up.weight", "offset": 1364256768, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.17.ffn_norm.weight", "offset": 1391781888, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 1391788032, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 1391789056, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.17.attn_output.weight", "offset": 1392575488, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.17.attn_q.bias", "offset": 1397294080, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 1397300224, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.17.attn_v.bias", "offset": 1402018816, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 1402019840, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.18.attn_norm.weight", "offset": 1402806272, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 1402812416, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.18.ffn_gate.weight", "offset": 1430337536, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.18.ffn_up.weight", "offset": 1457862656, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.18.ffn_norm.weight", "offset": 1485387776, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 1485393920, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 1485394944, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.18.attn_output.weight", "offset": 1486181376, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.18.attn_q.bias", "offset": 1490899968, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 1490906112, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.18.attn_v.bias", "offset": 1495624704, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 1495625728, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.19.attn_norm.weight", "offset": 1496412160, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 1496418304, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.19.ffn_gate.weight", "offset": 1523943424, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.19.ffn_up.weight", "offset": 1551468544, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.19.ffn_norm.weight", "offset": 1578993664, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 1578999808, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 1579000832, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.19.attn_output.weight", "offset": 1579787264, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.19.attn_q.bias", "offset": 1584505856, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 1584512000, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.19.attn_v.bias", "offset": 1589230592, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 1589231616, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 1590018048, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1590024192, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.2.ffn_gate.weight", "offset": 1617549312, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.2.ffn_up.weight", "offset": 1645074432, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.2.ffn_norm.weight", "offset": 1672599552, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 1672605696, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1672606720, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.2.attn_output.weight", "offset": 1673393152, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.2.attn_q.bias", "offset": 1678111744, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 1678117888, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.2.attn_v.bias", "offset": 1682836480, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 1682837504, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.20.attn_norm.weight", "offset": 1683623936, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 1683630080, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.20.ffn_gate.weight", "offset": 1711155200, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.20.ffn_up.weight", "offset": 1738680320, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.20.ffn_norm.weight", "offset": 1766205440, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 1766211584, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 1766212608, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.20.attn_output.weight", "offset": 1766999040, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.20.attn_q.bias", "offset": 1771717632, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 1771723776, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.20.attn_v.bias", "offset": 1776442368, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 1776443392, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 1777229824, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 1777235968, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.21.ffn_gate.weight", "offset": 1804761088, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.21.ffn_up.weight", "offset": 1832286208, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.21.ffn_norm.weight", "offset": 1859811328, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 1859817472, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 1859818496, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.21.attn_output.weight", "offset": 1860604928, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.21.attn_q.bias", "offset": 1865323520, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 1865329664, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.21.attn_v.bias", "offset": 1870048256, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 1870049280, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 1870835712, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 1870841856, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.22.ffn_gate.weight", "offset": 1898366976, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.22.ffn_up.weight", "offset": 1925892096, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.22.ffn_norm.weight", "offset": 1953417216, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 1953423360, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 1953424384, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.22.attn_output.weight", "offset": 1954210816, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.22.attn_q.bias", "offset": 1958929408, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 1958935552, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.22.attn_v.bias", "offset": 1963654144, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 1963655168, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 1964441600, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 1964447744, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.23.ffn_gate.weight", "offset": 1991972864, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.23.ffn_up.weight", "offset": 2019497984, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.23.ffn_norm.weight", "offset": 2047023104, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 2047029248, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2047030272, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.23.attn_output.weight", "offset": 2047816704, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.23.attn_q.bias", "offset": 2052535296, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 2052541440, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.23.attn_v.bias", "offset": 2057260032, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 2057261056, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 2058047488, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2058053632, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.24.ffn_gate.weight", "offset": 2085578752, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.24.ffn_up.weight", "offset": 2113103872, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.24.ffn_norm.weight", "offset": 2140628992, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 2140635136, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 2140636160, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.24.attn_output.weight", "offset": 2141422592, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.24.attn_q.bias", "offset": 2146141184, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 2146147328, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.24.attn_v.bias", "offset": 2150865920, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 2150866944, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.25.attn_norm.weight", "offset": 2151653376, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 2151659520, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.25.ffn_gate.weight", "offset": 2179184640, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.25.ffn_up.weight", "offset": 2206709760, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.25.ffn_norm.weight", "offset": 2234234880, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 2234241024, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 2234242048, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.25.attn_output.weight", "offset": 2235028480, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.25.attn_q.bias", "offset": 2239747072, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 2239753216, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.25.attn_v.bias", "offset": 2244471808, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 2244472832, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 2245259264, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 2245265408, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.26.ffn_gate.weight", "offset": 2272790528, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.26.ffn_up.weight", "offset": 2300315648, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.26.ffn_norm.weight", "offset": 2327840768, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 2327846912, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 2327847936, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.26.attn_output.weight", "offset": 2328634368, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.26.attn_q.bias", "offset": 2333352960, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 2333359104, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.26.attn_v.bias", "offset": 2338077696, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 2338078720, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.27.attn_norm.weight", "offset": 2338865152, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 2338871296, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.27.ffn_gate.weight", "offset": 2366396416, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.27.ffn_up.weight", "offset": 2393921536, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.27.ffn_norm.weight", "offset": 2421446656, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 2421452800, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 2421453824, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.27.attn_output.weight", "offset": 2422240256, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.27.attn_q.bias", "offset": 2426958848, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 2426964992, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.27.attn_v.bias", "offset": 2431683584, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 2431684608, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.3.attn_norm.weight", "offset": 2432471040, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "\u0000#\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0006\u0000\u0000\u0000\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0000", "offset": 2432477184, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.3.ffn_gate.weight", "offset": 2460002304, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.3.ffn_up.weight", "offset": 2487527424, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.3.ffn_norm.weight", "offset": 2515052544, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 2515058688, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 2515059712, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.3.attn_output.weight", "offset": 2515846144, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.3.attn_q.bias", "offset": 2520564736, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 2520570880, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.3.attn_v.bias", "offset": 2525289472, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 2525290496, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 2526076928, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 2526083072, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.4.ffn_gate.weight", "offset": 2553608192, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.4.ffn_up.weight", "offset": 2581133312, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.4.ffn_norm.weight", "offset": 2608658432, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 2608664576, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 2608665600, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.4.attn_output.weight", "offset": 2609452032, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.4.attn_q.bias", "offset": 2614170624, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 2614176768, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.4.attn_v.bias", "offset": 2618895360, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 2618896384, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.5.attn_norm.weight", "offset": 2619682816, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 2619688960, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.5.ffn_gate.weight", "offset": 2647214080, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.5.ffn_up.weight", "offset": 2674739200, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.5.ffn_norm.weight", "offset": 2702264320, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 2702270464, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 2702271488, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.5.attn_output.weight", "offset": 2703057920, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.5.attn_q.bias", "offset": 2707776512, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 2707782656, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.5.attn_v.bias", "offset": 2712501248, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 2712502272, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.6.attn_norm.weight", "offset": 2713288704, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2713294848, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.6.ffn_gate.weight", "offset": 2740819968, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.6.ffn_up.weight", "offset": 2768345088, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.6.ffn_norm.weight", "offset": 2795870208, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 2795876352, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 2795877376, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.6.attn_output.weight", "offset": 2796663808, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.6.attn_q.bias", "offset": 2801382400, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 2801388544, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.6.attn_v.bias", "offset": 2806107136, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 2806108160, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.7.attn_norm.weight", "offset": 2806894592, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2806900736, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.7.ffn_gate.weight", "offset": 2834425856, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.7.ffn_up.weight", "offset": 2861950976, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.7.ffn_norm.weight", "offset": 2889476096, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 2889482240, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2889483264, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.7.attn_output.weight", "offset": 2890269696, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.7.attn_q.bias", "offset": 2894988288, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 2894994432, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.7.attn_v.bias", "offset": 2899713024, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 2899714048, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.8.attn_norm.weight", "offset": 2900500480, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2900506624, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.8.ffn_gate.weight", "offset": 2928031744, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.8.ffn_up.weight", "offset": 2955556864, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.8.ffn_norm.weight", "offset": 2983081984, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 2983088128, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2983089152, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.8.attn_output.weight", "offset": 2983875584, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.8.attn_q.bias", "offset": 2988594176, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 2988600320, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.8.attn_v.bias", "offset": 2993318912, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 2993319936, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 2994106368, "shape": [ 1536 ], "size": 786432, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2994112512, "shape": [ 8960, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.9.ffn_gate.weight", "offset": 3021637632, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.9.ffn_up.weight", "offset": 3049162752, "shape": [ 1536, 8960 ], "size": 27525120, "type": 1 }, { "name": "blk.9.ffn_norm.weight", "offset": 3076687872, "shape": [ 1536 ], "size": 27525120, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 3076694016, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 3076695040, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "blk.9.attn_output.weight", "offset": 3077481472, "shape": [ 1536, 1536 ], "size": 786432, "type": 1 }, { "name": "blk.9.attn_q.bias", "offset": 3082200064, "shape": [ 1536 ], "size": 4718592, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 3082206208, "shape": [ 1536, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.9.attn_v.bias", "offset": 3086924800, "shape": [ 256 ], "size": 4718592, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 3086925824, "shape": [ 1536, 256 ], "size": 1024, "type": 1 }, { "name": "output_norm.weight", "offset": 3087712256, "shape": [ 1536 ], "size": 786432, "type": 0 } ], "version": 3 }