Meta Llama 3: The most capable openly available LLM to date

8B 70B

4M Pulls Updated 4 weeks ago

6a0746a1ec1a · 4.7GB
{ "metadata": { "general.architecture": "llama", "general.file_type": 2, "general.name": "Meta-Llama-3-8B-Instruct", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 32, "llama.context_length": 8192, "llama.embedding_length": 4096, "llama.feed_forward_length": 14336, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 500000, "llama.vocab_size": 128256, "tokenizer.ggml.bos_token_id": 128000, "tokenizer.ggml.eos_token_id": 128009, "tokenizer.ggml.merges": "... (280147 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.pre": "llama-bpe", "tokenizer.ggml.token_type": "... (128256 values)", "tokenizer.ggml.tokens": "... (128256 values)" }, "num_params": 8030261248, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 295501824, "shape": [ 4096 ], "size": 295501824, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 295518208, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.0.ffn_gate.weight", "offset": 328548352, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 361578496, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 394608640, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 394625024, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.0.attn_output.weight", "offset": 396984320, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.0.attn_q.weight", "offset": 406421504, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.0.attn_v.weight", "offset": 415858688, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 418217984, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 418234368, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.1.ffn_gate.weight", "offset": 451264512, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 484294656, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 517324800, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 517341184, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.1.attn_output.weight", "offset": 519700480, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.1.attn_q.weight", "offset": 529137664, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.1.attn_v.weight", "offset": 538574848, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 540934144, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 540950528, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.2.ffn_gate.weight", "offset": 573980672, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 607010816, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 640040960, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 640057344, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.2.attn_output.weight", "offset": 642416640, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.2.attn_q.weight", "offset": 651853824, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.2.attn_v.weight", "offset": 661291008, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 663650304, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 663666688, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.3.ffn_gate.weight", "offset": 696696832, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 729726976, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 762757120, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 762773504, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.3.attn_output.weight", "offset": 765132800, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.3.attn_q.weight", "offset": 774569984, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.3.attn_v.weight", "offset": 784007168, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 786366464, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 786382848, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.4.ffn_gate.weight", "offset": 819412992, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 852443136, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 885473280, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 885489664, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.4.attn_output.weight", "offset": 887848960, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.4.attn_q.weight", "offset": 897286144, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.4.attn_v.weight", "offset": 906723328, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 909082624, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 909099008, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.5.ffn_gate.weight", "offset": 942129152, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 975159296, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 1008189440, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1008205824, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.5.attn_output.weight", "offset": 1010565120, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.5.attn_q.weight", "offset": 1020002304, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.5.attn_v.weight", "offset": 1029439488, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 1031798784, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1031815168, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.6.ffn_gate.weight", "offset": 1064845312, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 1097875456, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 1130905600, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1130921984, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 1133281280, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.6.attn_q.weight", "offset": 1142718464, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.6.attn_v.weight", "offset": 1152155648, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.7.attn_norm.weight", "offset": 1154514944, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1154531328, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.7.ffn_gate.weight", "offset": 1187561472, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 1220591616, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 1253621760, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1253638144, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.7.attn_output.weight", "offset": 1255997440, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.7.attn_q.weight", "offset": 1265434624, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.7.attn_v.weight", "offset": 1274871808, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 1277231104, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1277247488, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.8.ffn_gate.weight", "offset": 1310277632, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 1343307776, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 1376337920, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1376354304, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.8.attn_output.weight", "offset": 1378713600, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.8.attn_q.weight", "offset": 1388150784, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.8.attn_v.weight", "offset": 1397587968, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 1399947264, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1399963648, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.10.ffn_gate.weight", "offset": 1432993792, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 1466023936, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 1499054080, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1499070464, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.10.attn_output.weight", "offset": 1501429760, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.10.attn_q.weight", "offset": 1510866944, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.10.attn_v.weight", "offset": 1520304128, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 1522663424, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1522679808, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.11.ffn_gate.weight", "offset": 1555709952, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 1588740096, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 1621770240, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1621786624, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.11.attn_output.weight", "offset": 1624145920, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.11.attn_q.weight", "offset": 1633583104, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.11.attn_v.weight", "offset": 1643020288, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 1645379584, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1645395968, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.12.ffn_gate.weight", "offset": 1678426112, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 1711456256, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 1744486400, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 1744502784, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.12.attn_output.weight", "offset": 1746862080, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.12.attn_q.weight", "offset": 1756299264, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.12.attn_v.weight", "offset": 1765736448, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 1768095744, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1768112128, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.13.ffn_gate.weight", "offset": 1801142272, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 1834172416, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 1867202560, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1867218944, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.13.attn_output.weight", "offset": 1869578240, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.13.attn_q.weight", "offset": 1879015424, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.13.attn_v.weight", "offset": 1888452608, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 1890811904, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1890828288, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.14.ffn_gate.weight", "offset": 1923858432, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 1956888576, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 1989918720, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1989935104, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.14.attn_output.weight", "offset": 1992294400, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.14.attn_q.weight", "offset": 2001731584, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.14.attn_v.weight", "offset": 2011168768, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 2013528064, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2013544448, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.15.ffn_gate.weight", "offset": 2046574592, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 2079604736, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 2112634880, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2112651264, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.15.attn_output.weight", "offset": 2115010560, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.15.attn_q.weight", "offset": 2124447744, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.15.attn_v.weight", "offset": 2133884928, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 2136244224, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2136260608, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.16.ffn_gate.weight", "offset": 2169290752, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 2202320896, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 2235351040, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2235367424, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.16.attn_output.weight", "offset": 2237726720, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.16.attn_q.weight", "offset": 2247163904, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.16.attn_v.weight", "offset": 2256601088, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 2258960384, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2258976768, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.17.ffn_gate.weight", "offset": 2292006912, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 2325037056, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 2358067200, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2358083584, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.17.attn_output.weight", "offset": 2360442880, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.17.attn_q.weight", "offset": 2369880064, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.17.attn_v.weight", "offset": 2379317248, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 2381676544, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2381692928, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.18.ffn_gate.weight", "offset": 2414723072, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 2447753216, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 2480783360, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2480799744, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.18.attn_output.weight", "offset": 2483159040, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.18.attn_q.weight", "offset": 2492596224, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.18.attn_v.weight", "offset": 2502033408, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.19.attn_norm.weight", "offset": 2504392704, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2504409088, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.19.ffn_gate.weight", "offset": 2537439232, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 2570469376, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.19.ffn_norm.weight", "offset": 2603499520, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2603515904, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.19.attn_output.weight", "offset": 2605875200, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.19.attn_q.weight", "offset": 2615312384, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.19.attn_v.weight", "offset": 2624749568, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.20.ffn_gate.weight", "offset": 2627108864, "shape": [ 4096, 14336 ], "size": 2359296, "type": 2 }, { "name": "blk.20.attn_k.weight", "offset": 2660139008, "shape": [ 4096, 1024 ], "size": 33030144, "type": 2 }, { "name": "blk.20.attn_output.weight", "offset": 2662498304, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.20.attn_q.weight", "offset": 2671935488, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.20.attn_v.weight", "offset": 2681372672, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 2683731968, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2683748352, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.9.ffn_gate.weight", "offset": 2716778496, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 2749808640, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 2782838784, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2782855168, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.9.attn_output.weight", "offset": 2785214464, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.9.attn_q.weight", "offset": 2794651648, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.9.attn_v.weight", "offset": 2804088832, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 2806448128, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2806464512, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 2839494656, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 2872524800, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.21.attn_norm.weight", "offset": 2872541184, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2872557568, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.21.ffn_gate.weight", "offset": 2905587712, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 2938617856, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 2971648000, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2971664384, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 2974023680, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.21.attn_q.weight", "offset": 2983460864, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.21.attn_v.weight", "offset": 2992898048, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.22.attn_norm.weight", "offset": 2995257344, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2995273728, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.22.ffn_gate.weight", "offset": 3028303872, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.22.ffn_up.weight", "offset": 3061334016, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 3094364160, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3094380544, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.22.attn_output.weight", "offset": 3096739840, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.22.attn_q.weight", "offset": 3106177024, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.22.attn_v.weight", "offset": 3115614208, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.23.attn_norm.weight", "offset": 3117973504, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3117989888, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.23.ffn_gate.weight", "offset": 3151020032, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 3184050176, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 3217080320, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3217096704, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.23.attn_output.weight", "offset": 3219456000, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.23.attn_q.weight", "offset": 3228893184, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.23.attn_v.weight", "offset": 3238330368, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.24.attn_norm.weight", "offset": 3240689664, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3240706048, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.24.ffn_gate.weight", "offset": 3273736192, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.24.ffn_up.weight", "offset": 3306766336, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.24.ffn_norm.weight", "offset": 3339796480, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 3339812864, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.24.attn_output.weight", "offset": 3342172160, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.24.attn_q.weight", "offset": 3351609344, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.24.attn_v.weight", "offset": 3361046528, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.25.attn_norm.weight", "offset": 3363405824, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3363422208, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.25.ffn_gate.weight", "offset": 3396452352, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.25.ffn_up.weight", "offset": 3429482496, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.25.ffn_norm.weight", "offset": 3462512640, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3462529024, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.25.attn_output.weight", "offset": 3464888320, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.25.attn_q.weight", "offset": 3474325504, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.25.attn_v.weight", "offset": 3483762688, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 3486121984, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3486138368, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.26.ffn_gate.weight", "offset": 3519168512, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.26.ffn_up.weight", "offset": 3552198656, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.26.ffn_norm.weight", "offset": 3585228800, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3585245184, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.26.attn_output.weight", "offset": 3587604480, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.26.attn_q.weight", "offset": 3597041664, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.26.attn_v.weight", "offset": 3606478848, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 3608838144, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3608854528, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.27.ffn_gate.weight", "offset": 3641884672, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.27.ffn_up.weight", "offset": 3674914816, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.27.ffn_norm.weight", "offset": 3707944960, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 3707961344, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.27.attn_output.weight", "offset": 3710320640, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.27.attn_q.weight", "offset": 3719757824, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.27.attn_v.weight", "offset": 3729195008, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.28.attn_norm.weight", "offset": 3731554304, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3731570688, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.28.ffn_gate.weight", "offset": 3764600832, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.28.ffn_up.weight", "offset": 3797630976, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.28.ffn_norm.weight", "offset": 3830661120, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3830677504, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.28.attn_output.weight", "offset": 3833036800, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.28.attn_q.weight", "offset": 3842473984, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.28.attn_v.weight", "offset": 3851911168, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.29.attn_norm.weight", "offset": 3854270464, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3854286848, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.29.ffn_gate.weight", "offset": 3887316992, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.29.ffn_up.weight", "offset": 3920347136, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.29.ffn_norm.weight", "offset": 3953377280, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 3953393664, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.29.attn_output.weight", "offset": 3955752960, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.29.attn_q.weight", "offset": 3965190144, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.29.attn_v.weight", "offset": 3974627328, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.30.attn_norm.weight", "offset": 3976986624, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3977003008, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.30.ffn_gate.weight", "offset": 4010033152, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.30.ffn_up.weight", "offset": 4043063296, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.30.ffn_norm.weight", "offset": 4076093440, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 4076109824, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.30.attn_output.weight", "offset": 4078469120, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.30.attn_q.weight", "offset": 4087906304, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.30.attn_v.weight", "offset": 4097343488, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.31.ffn_gate.weight", "offset": 4099702784, "shape": [ 4096, 14336 ], "size": 2359296, "type": 2 }, { "name": "blk.31.ffn_up.weight", "offset": 4132732928, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.31.attn_k.weight", "offset": 4165763072, "shape": [ 4096, 1024 ], "size": 33030144, "type": 2 }, { "name": "blk.31.attn_output.weight", "offset": 4168122368, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.31.attn_q.weight", "offset": 4177559552, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.31.attn_v.weight", "offset": 4186996736, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "output.weight", "offset": 4189356032, "shape": [ 4096, 128256 ], "size": 2359296, "type": 14 }, { "name": "blk.31.attn_norm.weight", "offset": 4620296192, "shape": [ 4096 ], "size": 430940160, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 4620312576, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.31.ffn_norm.weight", "offset": 4653342720, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "output_norm.weight", "offset": 4653359104, "shape": [ 4096 ], "size": 16384, "type": 0 } ], "version": 3 }