qwen

Qwen 1.5 is a series of large language models by Alibaba Cloud spanning from 0.5B to 110B parameters

0.5B 1.8B 4B 32B 72B 110B

1.9M Pulls Updated 11 days ago

379 Tags

... /
model
645fea8189fd · 5.8GB
{ "metadata": { "general.architecture": "qwen2", "general.file_type": 9, "general.name": "Qwen2-beta-7B", "general.quantization_version": 2, "qwen2.attention.head_count": 32, "qwen2.attention.head_count_kv": 32, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 32, "qwen2.context_length": 32768, "qwen2.embedding_length": 4096, "qwen2.feed_forward_length": 11008, "qwen2.use_parallel_residual": true, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 7721324544, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 466747392, "shape": [ 4096 ], "size": 466747392, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 466763776, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.0.ffn_gate.weight", "offset": 500580352, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 534396928, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 568213504, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 568229888, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 568246272, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.0.attn_output.weight", "offset": 580829184, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.0.attn_q.bias", "offset": 593412096, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 593428480, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.0.attn_v.bias", "offset": 606011392, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 606027776, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 618610688, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 618627072, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.1.ffn_gate.weight", "offset": 652443648, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.1.ffn_up.weight", "offset": 686260224, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.1.ffn_norm.weight", "offset": 720076800, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 720093184, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 720109568, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.1.attn_output.weight", "offset": 732692480, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.1.attn_q.bias", "offset": 745275392, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 745291776, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.1.attn_v.bias", "offset": 757874688, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 757891072, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 770473984, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 770490368, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.2.ffn_gate.weight", "offset": 804306944, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.2.ffn_up.weight", "offset": 838123520, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.2.ffn_norm.weight", "offset": 871940096, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 871956480, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 871972864, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.2.attn_output.weight", "offset": 884555776, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.2.attn_q.bias", "offset": 897138688, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 897155072, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.2.attn_v.bias", "offset": 909737984, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 909754368, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 922337280, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 922353664, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.3.ffn_gate.weight", "offset": 956170240, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.3.ffn_up.weight", "offset": 989986816, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.3.ffn_norm.weight", "offset": 1023803392, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 1023819776, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1023836160, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.3.attn_output.weight", "offset": 1036419072, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.3.attn_q.bias", "offset": 1049001984, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 1049018368, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.3.attn_v.bias", "offset": 1061601280, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 1061617664, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 1074200576, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1074216960, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.4.ffn_gate.weight", "offset": 1108033536, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.4.ffn_up.weight", "offset": 1141850112, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.4.ffn_norm.weight", "offset": 1175666688, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 1175683072, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1175699456, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.4.attn_output.weight", "offset": 1188282368, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.4.attn_q.bias", "offset": 1200865280, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 1200881664, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.4.attn_v.bias", "offset": 1213464576, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 1213480960, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 1226063872, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1226080256, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.5.ffn_gate.weight", "offset": 1259896832, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.5.ffn_up.weight", "offset": 1293713408, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.5.ffn_norm.weight", "offset": 1327529984, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 1327546368, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1327562752, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.5.attn_output.weight", "offset": 1340145664, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.5.attn_q.bias", "offset": 1352728576, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1352744960, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.5.attn_v.bias", "offset": 1365327872, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 1365344256, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.6.ffn_gate.weight", "offset": 1377927168, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.6.ffn_up.weight", "offset": 1411743744, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.6.attn_k.bias", "offset": 1445560320, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1445576704, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 1458159616, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.6.attn_q.bias", "offset": 1470742528, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1470758912, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.6.attn_v.bias", "offset": 1483341824, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 1483358208, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 1495941120, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1495957504, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.10.ffn_gate.weight", "offset": 1529774080, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.10.ffn_up.weight", "offset": 1563590656, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 1597407232, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 1597423616, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1597440000, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.10.attn_output.weight", "offset": 1610022912, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.10.attn_q.bias", "offset": 1622605824, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1622622208, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.10.attn_v.bias", "offset": 1635205120, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 1635221504, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 1647804416, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1647820800, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.11.ffn_gate.weight", "offset": 1681637376, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.11.ffn_up.weight", "offset": 1715453952, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.11.ffn_norm.weight", "offset": 1749270528, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 1749286912, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1749303296, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.11.attn_output.weight", "offset": 1761886208, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.11.attn_q.bias", "offset": 1774469120, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1774485504, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.11.attn_v.bias", "offset": 1787068416, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 1787084800, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 1799667712, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1799684096, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.12.ffn_gate.weight", "offset": 1833500672, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.12.ffn_up.weight", "offset": 1867317248, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 1901133824, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 1901150208, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 1901166592, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.12.attn_output.weight", "offset": 1913749504, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.12.attn_q.bias", "offset": 1926332416, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1926348800, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.12.attn_v.bias", "offset": 1938931712, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 1938948096, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 1951531008, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1951547392, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.13.ffn_gate.weight", "offset": 1985363968, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.13.ffn_up.weight", "offset": 2019180544, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 2052997120, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 2053013504, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2053029888, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.13.attn_output.weight", "offset": 2065612800, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.13.attn_q.bias", "offset": 2078195712, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 2078212096, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.13.attn_v.bias", "offset": 2090795008, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 2090811392, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 2103394304, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2103410688, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.14.ffn_gate.weight", "offset": 2137227264, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.14.ffn_up.weight", "offset": 2171043840, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 2204860416, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 2204876800, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2204893184, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.14.attn_output.weight", "offset": 2217476096, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.14.attn_q.bias", "offset": 2230059008, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 2230075392, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.14.attn_v.bias", "offset": 2242658304, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 2242674688, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 2255257600, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2255273984, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.15.ffn_gate.weight", "offset": 2289090560, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.15.ffn_up.weight", "offset": 2322907136, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.15.ffn_norm.weight", "offset": 2356723712, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 2356740096, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2356756480, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.15.attn_output.weight", "offset": 2369339392, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.15.attn_q.bias", "offset": 2381922304, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 2381938688, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.15.attn_v.bias", "offset": 2394521600, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 2394537984, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.16.ffn_gate.weight", "offset": 2407120896, "shape": [ 4096, 11008 ], "size": 12582912, "type": 7 }, { "name": "blk.16.attn_k.bias", "offset": 2440937472, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2440953856, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.16.attn_output.weight", "offset": 2453536768, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.16.attn_q.bias", "offset": 2466119680, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 2466136064, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.16.attn_v.bias", "offset": 2478718976, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 2478735360, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 2491318272, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2491334656, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.6.ffn_norm.weight", "offset": 2525151232, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2525167616, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2525184000, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.7.ffn_gate.weight", "offset": 2559000576, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 2592817152, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.7.ffn_norm.weight", "offset": 2626633728, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 2626650112, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2626666496, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.7.attn_output.weight", "offset": 2639249408, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.7.attn_q.bias", "offset": 2651832320, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 2651848704, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.7.attn_v.bias", "offset": 2664431616, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 2664448000, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.8.attn_norm.weight", "offset": 2677030912, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2677047296, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.8.ffn_gate.weight", "offset": 2710863872, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.8.ffn_up.weight", "offset": 2744680448, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.8.ffn_norm.weight", "offset": 2778497024, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 2778513408, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2778529792, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.8.attn_output.weight", "offset": 2791112704, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.8.attn_q.bias", "offset": 2803695616, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 2803712000, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.8.attn_v.bias", "offset": 2816294912, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 2816311296, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 2828894208, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2828910592, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.9.ffn_gate.weight", "offset": 2862727168, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.9.ffn_up.weight", "offset": 2896543744, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 2930360320, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 2930376704, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2930393088, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.9.attn_output.weight", "offset": 2942976000, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.9.attn_q.bias", "offset": 2955558912, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 2955575296, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.9.attn_v.bias", "offset": 2968158208, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 2968174592, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 2980757504, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2980773888, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 3014590464, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 3048407040, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.17.attn_norm.weight", "offset": 3048423424, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3048439808, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.17.ffn_gate.weight", "offset": 3082256384, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.17.ffn_up.weight", "offset": 3116072960, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.17.ffn_norm.weight", "offset": 3149889536, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 3149905920, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 3149922304, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.17.attn_output.weight", "offset": 3162505216, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.17.attn_q.bias", "offset": 3175088128, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 3175104512, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.17.attn_v.bias", "offset": 3187687424, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 3187703808, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 3200286720, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3200303104, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.18.ffn_gate.weight", "offset": 3234119680, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.18.ffn_up.weight", "offset": 3267936256, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 3301752832, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 3301769216, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 3301785600, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.18.attn_output.weight", "offset": 3314368512, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.18.attn_q.bias", "offset": 3326951424, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 3326967808, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.18.attn_v.bias", "offset": 3339550720, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 3339567104, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.19.attn_norm.weight", "offset": 3352150016, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3352166400, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.19.ffn_gate.weight", "offset": 3385982976, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.19.ffn_up.weight", "offset": 3419799552, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.19.ffn_norm.weight", "offset": 3453616128, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 3453632512, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 3453648896, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.19.attn_output.weight", "offset": 3466231808, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.19.attn_q.bias", "offset": 3478814720, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 3478831104, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.19.attn_v.bias", "offset": 3491414016, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 3491430400, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 3504013312, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3504029696, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.20.ffn_gate.weight", "offset": 3537846272, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.20.ffn_up.weight", "offset": 3571662848, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.20.ffn_norm.weight", "offset": 3605479424, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 3605495808, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 3605512192, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.20.attn_output.weight", "offset": 3618095104, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.20.attn_q.bias", "offset": 3630678016, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 3630694400, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.20.attn_v.bias", "offset": 3643277312, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 3643293696, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 3655876608, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3655892992, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.21.ffn_gate.weight", "offset": 3689709568, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.21.ffn_up.weight", "offset": 3723526144, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.21.ffn_norm.weight", "offset": 3757342720, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 3757359104, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 3757375488, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 3769958400, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.21.attn_q.bias", "offset": 3782541312, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 3782557696, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.21.attn_v.bias", "offset": 3795140608, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 3795156992, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.22.attn_norm.weight", "offset": 3807739904, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3807756288, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.22.ffn_gate.weight", "offset": 3841572864, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.22.ffn_up.weight", "offset": 3875389440, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.22.ffn_norm.weight", "offset": 3909206016, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 3909222400, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3909238784, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.22.attn_output.weight", "offset": 3921821696, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.22.attn_q.bias", "offset": 3934404608, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 3934420992, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.22.attn_v.bias", "offset": 3947003904, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 3947020288, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 3959603200, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3959619584, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.23.ffn_gate.weight", "offset": 3993436160, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.23.ffn_up.weight", "offset": 4027252736, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.23.ffn_norm.weight", "offset": 4061069312, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 4061085696, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 4061102080, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.23.attn_output.weight", "offset": 4073684992, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.23.attn_q.bias", "offset": 4086267904, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 4086284288, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.23.attn_v.bias", "offset": 4098867200, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 4098883584, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.24.attn_norm.weight", "offset": 4111466496, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4111482880, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.24.ffn_gate.weight", "offset": 4145299456, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.24.ffn_up.weight", "offset": 4179116032, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.24.ffn_norm.weight", "offset": 4212932608, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 4212948992, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 4212965376, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.24.attn_output.weight", "offset": 4225548288, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.24.attn_q.bias", "offset": 4238131200, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 4238147584, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.24.attn_v.bias", "offset": 4250730496, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 4250746880, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.25.attn_norm.weight", "offset": 4263329792, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4263346176, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.25.ffn_gate.weight", "offset": 4297162752, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.25.ffn_up.weight", "offset": 4330979328, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.25.ffn_norm.weight", "offset": 4364795904, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 4364812288, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 4364828672, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.25.attn_output.weight", "offset": 4377411584, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.25.attn_q.bias", "offset": 4389994496, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 4390010880, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.25.attn_v.bias", "offset": 4402593792, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 4402610176, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.26.attn_k.bias", "offset": 4415193088, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 4415209472, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.26.attn_output.weight", "offset": 4427792384, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.26.attn_q.bias", "offset": 4440375296, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 4440391680, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.26.attn_v.bias", "offset": 4452974592, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 4452990976, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "output.weight", "offset": 4465573888, "shape": [ 4096, 151936 ], "size": 12582912, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 4976078848, "shape": [ 4096 ], "size": 510504960, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4976095232, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.26.ffn_gate.weight", "offset": 5009911808, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.26.ffn_up.weight", "offset": 5043728384, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.26.ffn_norm.weight", "offset": 5077544960, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.27.attn_norm.weight", "offset": 5077561344, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5077577728, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.27.ffn_gate.weight", "offset": 5111394304, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.27.ffn_up.weight", "offset": 5145210880, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.27.ffn_norm.weight", "offset": 5179027456, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 5179043840, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 5179060224, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.27.attn_output.weight", "offset": 5191643136, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.27.attn_q.bias", "offset": 5204226048, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 5204242432, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.27.attn_v.bias", "offset": 5216825344, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 5216841728, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.28.attn_norm.weight", "offset": 5229424640, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5229441024, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.28.ffn_gate.weight", "offset": 5263257600, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.28.ffn_up.weight", "offset": 5297074176, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.28.ffn_norm.weight", "offset": 5330890752, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.28.attn_k.bias", "offset": 5330907136, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 5330923520, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.28.attn_output.weight", "offset": 5343506432, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.28.attn_q.bias", "offset": 5356089344, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 5356105728, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.28.attn_v.bias", "offset": 5368688640, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.28.attn_v.weight", "offset": 5368705024, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.29.attn_norm.weight", "offset": 5381287936, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 5381304320, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.29.ffn_gate.weight", "offset": 5415120896, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.29.ffn_up.weight", "offset": 5448937472, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.29.ffn_norm.weight", "offset": 5482754048, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.29.attn_k.bias", "offset": 5482770432, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 5482786816, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.29.attn_output.weight", "offset": 5495369728, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.29.attn_q.bias", "offset": 5507952640, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 5507969024, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.29.attn_v.bias", "offset": 5520551936, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.29.attn_v.weight", "offset": 5520568320, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.30.attn_norm.weight", "offset": 5533151232, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 5533167616, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.30.ffn_gate.weight", "offset": 5566984192, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.30.ffn_up.weight", "offset": 5600800768, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.30.ffn_norm.weight", "offset": 5634617344, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.30.attn_k.bias", "offset": 5634633728, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 5634650112, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.30.attn_output.weight", "offset": 5647233024, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.30.attn_q.bias", "offset": 5659815936, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 5659832320, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.30.attn_v.bias", "offset": 5672415232, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.30.attn_v.weight", "offset": 5672431616, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.31.attn_norm.weight", "offset": 5685014528, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 5685030912, "shape": [ 11008, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.31.ffn_gate.weight", "offset": 5718847488, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.31.ffn_up.weight", "offset": 5752664064, "shape": [ 4096, 11008 ], "size": 33816576, "type": 7 }, { "name": "blk.31.ffn_norm.weight", "offset": 5786480640, "shape": [ 4096 ], "size": 33816576, "type": 0 }, { "name": "blk.31.attn_k.bias", "offset": 5786497024, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 5786513408, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.31.attn_output.weight", "offset": 5799096320, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.31.attn_q.bias", "offset": 5811679232, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 5811695616, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "blk.31.attn_v.bias", "offset": 5824278528, "shape": [ 4096 ], "size": 12582912, "type": 0 }, { "name": "blk.31.attn_v.weight", "offset": 5824294912, "shape": [ 4096, 4096 ], "size": 16384, "type": 7 }, { "name": "output_norm.weight", "offset": 5836877824, "shape": [ 4096 ], "size": 12582912, "type": 0 } ], "version": 3 }