wangshenzhi / llama3-8b-chinese-chat-ollama-q8

{ "metadata": { "general.architecture": "llama", "general.file_type": 7, "general.name": "outputs", "llama.attention.head_count": 32, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 32, "llama.context_length": 8192, "llama.embedding_length": 4096, "llama.feed_forward_length": 14336, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 500000, "llama.vocab_size": 128256, "tokenizer.ggml.bos_token_id": 128000, "tokenizer.ggml.eos_token_id": 128009, "tokenizer.ggml.merges": "... (280147 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 128009, "tokenizer.ggml.scores": "... (128256 values)", "tokenizer.ggml.token_type": "... (128256 values)", "tokenizer.ggml.tokens": "... (128256 values)" }, "num_params": 8030261248, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 558170112, "shape": [ 4096 ], "size": 558170112, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 558186496, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.0.ffn_gate.weight", "offset": 620576768, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 682967040, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 745357312, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 745373696, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 749830144, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.0.attn_q.weight", "offset": 767655936, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.0.attn_v.weight", "offset": 785481728, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 789938176, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 789954560, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.1.ffn_gate.weight", "offset": 852344832, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.1.ffn_up.weight", "offset": 914735104, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.1.ffn_norm.weight", "offset": 977125376, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 977141760, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 981598208, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.1.attn_q.weight", "offset": 999424000, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.1.attn_v.weight", "offset": 1017249792, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 1021706240, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1021722624, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.2.ffn_gate.weight", "offset": 1084112896, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.2.ffn_up.weight", "offset": 1146503168, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.2.ffn_norm.weight", "offset": 1208893440, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1208909824, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 1213366272, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.2.attn_q.weight", "offset": 1231192064, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.2.attn_v.weight", "offset": 1249017856, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.3.attn_norm.weight", "offset": 1253474304, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1253490688, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.3.ffn_gate.weight", "offset": 1315880960, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.3.ffn_up.weight", "offset": 1378271232, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.3.ffn_norm.weight", "offset": 1440661504, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1440677888, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 1445134336, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.3.attn_q.weight", "offset": 1462960128, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.3.attn_v.weight", "offset": 1480785920, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 1485242368, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1485258752, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.4.ffn_gate.weight", "offset": 1547649024, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.4.ffn_up.weight", "offset": 1610039296, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.4.ffn_norm.weight", "offset": 1672429568, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1672445952, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 1676902400, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.4.attn_q.weight", "offset": 1694728192, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.4.attn_v.weight", "offset": 1712553984, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.5.attn_norm.weight", "offset": 1717010432, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1717026816, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.5.ffn_gate.weight", "offset": 1779417088, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.5.ffn_up.weight", "offset": 1841807360, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.5.ffn_norm.weight", "offset": 1904197632, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1904214016, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 1908670464, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.5.attn_q.weight", "offset": 1926496256, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.5.attn_v.weight", "offset": 1944322048, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.6.attn_norm.weight", "offset": 1948778496, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1948794880, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.6.ffn_gate.weight", "offset": 2011185152, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.6.ffn_up.weight", "offset": 2073575424, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.6.ffn_norm.weight", "offset": 2135965696, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 2135982080, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 2140438528, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.6.attn_q.weight", "offset": 2158264320, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.6.attn_v.weight", "offset": 2176090112, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.7.attn_norm.weight", "offset": 2180546560, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2180562944, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.7.ffn_gate.weight", "offset": 2242953216, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 2305343488, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.7.ffn_norm.weight", "offset": 2367733760, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2367750144, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 2372206592, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.7.attn_q.weight", "offset": 2390032384, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.7.attn_v.weight", "offset": 2407858176, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.8.attn_norm.weight", "offset": 2412314624, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2412331008, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.8.ffn_gate.weight", "offset": 2474721280, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.8.ffn_up.weight", "offset": 2537111552, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.8.ffn_norm.weight", "offset": 2599501824, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2599518208, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 2603974656, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.8.attn_q.weight", "offset": 2621800448, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.8.attn_v.weight", "offset": 2639626240, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 2644082688, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 2644099072, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.10.ffn_gate.weight", "offset": 2706489344, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.10.ffn_up.weight", "offset": 2768879616, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.10.ffn_norm.weight", "offset": 2831269888, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 2831286272, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 2835742720, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.10.attn_q.weight", "offset": 2853568512, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.10.attn_v.weight", "offset": 2871394304, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 2875850752, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 2875867136, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.11.ffn_gate.weight", "offset": 2938257408, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.11.ffn_up.weight", "offset": 3000647680, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.11.ffn_norm.weight", "offset": 3063037952, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 3063054336, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 3067510784, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.11.attn_q.weight", "offset": 3085336576, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.11.attn_v.weight", "offset": 3103162368, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.12.attn_norm.weight", "offset": 3107618816, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 3107635200, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.12.ffn_gate.weight", "offset": 3170025472, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.12.ffn_up.weight", "offset": 3232415744, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.12.ffn_norm.weight", "offset": 3294806016, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 3294822400, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 3299278848, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.12.attn_q.weight", "offset": 3317104640, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.12.attn_v.weight", "offset": 3334930432, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.13.attn_norm.weight", "offset": 3339386880, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 3339403264, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.13.ffn_gate.weight", "offset": 3401793536, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.13.ffn_up.weight", "offset": 3464183808, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.13.ffn_norm.weight", "offset": 3526574080, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 3526590464, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 3531046912, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.13.attn_q.weight", "offset": 3548872704, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.13.attn_v.weight", "offset": 3566698496, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 3571154944, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3571171328, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.14.ffn_gate.weight", "offset": 3633561600, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.14.ffn_up.weight", "offset": 3695951872, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.14.ffn_norm.weight", "offset": 3758342144, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 3758358528, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 3762814976, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.14.attn_q.weight", "offset": 3780640768, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.14.attn_v.weight", "offset": 3798466560, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "e.weight\u0002\u0000\u0000\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\u00008\u0000", "offset": 3802923008, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3802939392, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.15.ffn_gate.weight", "offset": 3865329664, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.15.ffn_up.weight", "offset": 3927719936, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.15.ffn_norm.weight", "offset": 3990110208, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 3990126592, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 3994583040, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.15.attn_q.weight", "offset": 4012408832, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.15.attn_v.weight", "offset": 4030234624, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.16.attn_norm.weight", "offset": 4034691072, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 4034707456, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.16.ffn_gate.weight", "offset": 4097097728, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.16.ffn_up.weight", "offset": 4159488000, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.16.ffn_norm.weight", "offset": 4221878272, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 4221894656, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 4226351104, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.16.attn_q.weight", "offset": 4244176896, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.16.attn_v.weight", "offset": 4262002688, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 4266459136, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4266475520, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.17.ffn_gate.weight", "offset": 4328865792, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.17.ffn_up.weight", "offset": 4391256064, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.17.ffn_norm.weight", "offset": 4453646336, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 4453662720, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 4458119168, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.17.attn_q.weight", "offset": 4475944960, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.17.attn_v.weight", "offset": 4493770752, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.18.attn_norm.weight", "offset": 4498227200, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 4498243584, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.18.ffn_gate.weight", "offset": 4560633856, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.18.ffn_up.weight", "offset": 4623024128, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 4685414400, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 4685430784, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 4689887232, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.18.attn_q.weight", "offset": 4707713024, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.18.attn_v.weight", "offset": 4725538816, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.19.attn_norm.weight", "offset": 4729995264, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4730011648, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.19.ffn_gate.weight", "offset": 4792401920, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.19.ffn_up.weight", "offset": 4854792192, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.19.ffn_norm.weight", "offset": 4917182464, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 4917198848, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 4921655296, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.19.attn_q.weight", "offset": 4939481088, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.19.attn_v.weight", "offset": 4957306880, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.20.ffn_gate.weight", "offset": 4961763328, "shape": [ 4096, 14336 ], "size": 4456448, "type": 8 }, { "name": "blk.20.attn_k.weight", "offset": 5024153600, "shape": [ 4096, 1024 ], "size": 62390272, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 5028610048, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.20.attn_q.weight", "offset": 5046435840, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.20.attn_v.weight", "offset": 5064261632, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 5068718080, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 5068734464, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.9.ffn_gate.weight", "offset": 5131124736, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.9.ffn_up.weight", "offset": 5193515008, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.9.ffn_norm.weight", "offset": 5255905280, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 5255921664, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 5260378112, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.9.attn_q.weight", "offset": 5278203904, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.9.attn_v.weight", "offset": 5296029696, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.20.attn_norm.weight", "offset": 5300486144, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 5300502528, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.20.ffn_up.weight", "offset": 5362892800, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.20.ffn_norm.weight", "offset": 5425283072, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.21.attn_norm.weight", "offset": 5425299456, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5425315840, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.21.ffn_gate.weight", "offset": 5487706112, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.21.ffn_up.weight", "offset": 5550096384, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.21.ffn_norm.weight", "offset": 5612486656, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 5612503040, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 5616959488, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.21.attn_q.weight", "offset": 5634785280, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.21.attn_v.weight", "offset": 5652611072, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 5657067520, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5657083904, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.22.ffn_gate.weight", "offset": 5719474176, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.22.ffn_up.weight", "offset": 5781864448, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.22.ffn_norm.weight", "offset": 5844254720, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 5844271104, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 5848727552, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.22.attn_q.weight", "offset": 5866553344, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.22.attn_v.weight", "offset": 5884379136, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.23.attn_norm.weight", "offset": 5888835584, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5888851968, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.23.ffn_gate.weight", "offset": 5951242240, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.23.ffn_up.weight", "offset": 6013632512, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.23.ffn_norm.weight", "offset": 6076022784, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 6076039168, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 6080495616, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.23.attn_q.weight", "offset": 6098321408, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.23.attn_v.weight", "offset": 6116147200, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.24.attn_norm.weight", "offset": 6120603648, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 6120620032, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.24.ffn_gate.weight", "offset": 6183010304, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.24.ffn_up.weight", "offset": 6245400576, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.24.ffn_norm.weight", "offset": 6307790848, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 6307807232, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 6312263680, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.24.attn_q.weight", "offset": 6330089472, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.24.attn_v.weight", "offset": 6347915264, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.25.attn_norm.weight", "offset": 6352371712, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 6352388096, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.25.ffn_gate.weight", "offset": 6414778368, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.25.ffn_up.weight", "offset": 6477168640, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.25.ffn_norm.weight", "offset": 6539558912, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 6539575296, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 6544031744, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.25.attn_q.weight", "offset": 6561857536, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.25.attn_v.weight", "offset": 6579683328, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.26.attn_norm.weight", "offset": 6584139776, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 6584156160, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.26.ffn_gate.weight", "offset": 6646546432, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.26.ffn_up.weight", "offset": 6708936704, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.26.ffn_norm.weight", "offset": 6771326976, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 6771343360, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 6775799808, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.26.attn_q.weight", "offset": 6793625600, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.26.attn_v.weight", "offset": 6811451392, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.27.attn_norm.weight", "offset": 6815907840, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 6815924224, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.27.ffn_gate.weight", "offset": 6878314496, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.27.ffn_up.weight", "offset": 6940704768, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.27.ffn_norm.weight", "offset": 7003095040, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 7003111424, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 7007567872, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.27.attn_q.weight", "offset": 7025393664, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.27.attn_v.weight", "offset": 7043219456, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.28.attn_norm.weight", "offset": 7047675904, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 7047692288, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.28.ffn_gate.weight", "offset": 7110082560, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.28.ffn_up.weight", "offset": 7172472832, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.28.ffn_norm.weight", "offset": 7234863104, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 7234879488, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 7239335936, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.28.attn_q.weight", "offset": 7257161728, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.28.attn_v.weight", "offset": 7274987520, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.29.attn_norm.weight", "offset": 7279443968, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 7279460352, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.29.ffn_gate.weight", "offset": 7341850624, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.29.ffn_up.weight", "offset": 7404240896, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.29.ffn_norm.weight", "offset": 7466631168, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 7466647552, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 7471104000, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.29.attn_q.weight", "offset": 7488929792, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.29.attn_v.weight", "offset": 7506755584, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.30.attn_norm.weight", "offset": 7511212032, "shape": [ 4096 ], "size": 4456448, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 7511228416, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.30.ffn_gate.weight", "offset": 7573618688, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.30.ffn_up.weight", "offset": 7636008960, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.30.ffn_norm.weight", "offset": 7698399232, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 7698415616, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 7702872064, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.30.attn_q.weight", "offset": 7720697856, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.30.attn_v.weight", "offset": 7738523648, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.31.ffn_gate.weight", "offset": 7742980096, "shape": [ 4096, 14336 ], "size": 4456448, "type": 8 }, { "name": "blk.31.ffn_up.weight", "offset": 7805370368, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.31.attn_k.weight", "offset": 7867760640, "shape": [ 4096, 1024 ], "size": 62390272, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 7872217088, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.31.attn_q.weight", "offset": 7890042880, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.31.attn_v.weight", "offset": 7907868672, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "output.weight", "offset": 7912325120, "shape": [ 4096, 128256 ], "size": 4456448, "type": 8 }, { "name": "blk.31.attn_norm.weight", "offset": 8470495232, "shape": [ 4096 ], "size": 558170112, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 8470511616, "shape": [ 14336, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.31.ffn_norm.weight", "offset": 8532901888, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "output_norm.weight", "offset": 8532918272, "shape": [ 4096 ], "size": 16384, "type": 0 } ], "version": 3 }