codellama

{ "metadata": { "general.architecture": "llama", "general.file_type": 10, "general.name": "codellama", "general.quantization_version": 2, "llama.attention.head_count": 40, "llama.attention.head_count_kv": 40, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 40, "llama.context_length": 16384, "llama.embedding_length": 5120, "llama.feed_forward_length": 13824, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32000 values)", "tokenizer.ggml.token_type": "... (32000 values)", "tokenizer.ggml.tokens": "... (32000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13015864320, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 53760000, "shape": [ 5120 ], "size": 53760000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 53780480, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 84193280, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 114606080, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 145018880, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 145039360, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.0.attn_output.weight", "offset": 153640960, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.0.attn_q.weight", "offset": 164904960, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.0.attn_v.weight", "offset": 173506560, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 184770560, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 184791040, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.1.ffn_gate.weight", "offset": 215203840, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 245616640, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 276029440, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 276049920, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.1.attn_output.weight", "offset": 284651520, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.1.attn_q.weight", "offset": 295915520, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.1.attn_v.weight", "offset": 304517120, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.10.attn_norm.weight", "offset": 315781120, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 315801600, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.10.ffn_gate.weight", "offset": 346214400, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 376627200, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 407040000, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 407060480, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.10.attn_output.weight", "offset": 415662080, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.10.attn_q.weight", "offset": 426926080, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.10.attn_v.weight", "offset": 435527680, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 446791680, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 446812160, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.11.ffn_gate.weight", "offset": 477224960, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 507637760, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 538050560, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 538071040, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.11.attn_output.weight", "offset": 546672640, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.11.attn_q.weight", "offset": 557936640, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.11.attn_v.weight", "offset": 566538240, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 577802240, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 577822720, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 608235520, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 638648320, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 669061120, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 669081600, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.12.attn_output.weight", "offset": 677683200, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.12.attn_q.weight", "offset": 688947200, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.12.attn_v.weight", "offset": 697548800, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 708812800, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 708833280, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 739246080, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 769658880, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 800071680, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 800092160, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.13.attn_output.weight", "offset": 808693760, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.13.attn_q.weight", "offset": 819957760, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.13.attn_v.weight", "offset": 828559360, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 839823360, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 839843840, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.14.ffn_gate.weight", "offset": 870256640, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 900669440, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 931082240, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 931102720, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.14.attn_output.weight", "offset": 939704320, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.14.attn_q.weight", "offset": 950968320, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.14.attn_v.weight", "offset": 959569920, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.15.attn_k.weight", "offset": 970833920, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.15.attn_q.weight", "offset": 979435520, "shape": [ 5120, 5120 ], "size": 8601600, "type": 10 }, { "name": "blk.2.attn_norm.weight", "offset": 988037120, "shape": [ 5120 ], "size": 8601600, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 988057600, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.2.ffn_gate.weight", "offset": 1018470400, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 1048883200, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 1079296000, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1079316480, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.2.attn_output.weight", "offset": 1087918080, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.2.attn_q.weight", "offset": 1099182080, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.2.attn_v.weight", "offset": 1107783680, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 1119047680, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1119068160, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.3.ffn_gate.weight", "offset": 1149480960, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 1179893760, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 1210306560, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1210327040, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.3.attn_output.weight", "offset": 1218928640, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.3.attn_q.weight", "offset": 1230192640, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.3.attn_v.weight", "offset": 1238794240, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 1250058240, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1250078720, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.4.ffn_gate.weight", "offset": 1280491520, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 1310904320, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 1341317120, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1341337600, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.4.attn_output.weight", "offset": 1349939200, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.4.attn_q.weight", "offset": 1361203200, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.4.attn_v.weight", "offset": 1369804800, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 1381068800, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1381089280, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.5.ffn_gate.weight", "offset": 1411502080, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 1441914880, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 1472327680, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1472348160, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.5.attn_output.weight", "offset": 1480949760, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.5.attn_q.weight", "offset": 1492213760, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.5.attn_v.weight", "offset": 1500815360, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 1512079360, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1512099840, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.6.ffn_gate.weight", "offset": 1542512640, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 1572925440, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 1603338240, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1603358720, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.6.attn_output.weight", "offset": 1611960320, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.6.attn_q.weight", "offset": 1623224320, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.6.attn_v.weight", "offset": 1631825920, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.7.attn_norm.weight", "offset": 1643089920, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1643110400, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.7.ffn_gate.weight", "offset": 1673523200, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 1703936000, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 1734348800, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1734369280, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.7.attn_output.weight", "offset": 1742970880, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.7.attn_q.weight", "offset": 1754234880, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.7.attn_v.weight", "offset": 1762836480, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 1774100480, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1774120960, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.8.ffn_gate.weight", "offset": 1804533760, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 1834946560, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 1865359360, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1865379840, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.8.attn_output.weight", "offset": 1873981440, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.8.attn_q.weight", "offset": 1885245440, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.8.attn_v.weight", "offset": 1893847040, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 1905111040, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1905131520, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.9.ffn_gate.weight", "offset": 1935544320, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 1965957120, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 1996369920, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1996390400, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.9.attn_output.weight", "offset": 2004992000, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.9.attn_q.weight", "offset": 2016256000, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.9.attn_v.weight", "offset": 2024857600, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 2036121600, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2036142080, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.15.ffn_gate.weight", "offset": 2066554880, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 2096967680, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 2127380480, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 2127400960, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 2138664960, "shape": [ 5120, 5120 ], "size": 11264000, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 2149928960, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2149949440, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 2180362240, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 2210775040, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 2241187840, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2241208320, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.16.attn_output.weight", "offset": 2249809920, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.16.attn_q.weight", "offset": 2261073920, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.16.attn_v.weight", "offset": 2269675520, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 2280939520, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2280960000, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 2311372800, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 2341785600, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "\u0000\u0000\u0000\u0000\u0000\ufffdd\ufffd\u0000\u0000\u0000\u0000\u0014\u0000\u0000\u0000\u0000\u0000\u0000\u0000bl", "offset": 2372198400, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2372218880, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.17.attn_output.weight", "offset": 2380820480, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.17.attn_q.weight", "offset": 2392084480, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.17.attn_v.weight", "offset": 2400686080, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 2411950080, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2411970560, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.18.ffn_gate.weight", "offset": 2442383360, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 2472796160, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 2503208960, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2503229440, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.18.attn_output.weight", "offset": 2511831040, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.18.attn_q.weight", "offset": 2523095040, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.18.attn_v.weight", "offset": 2531696640, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 2542960640, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2542981120, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.19.ffn_gate.weight", "offset": 2573393920, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 2603806720, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 2634219520, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2634240000, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.19.attn_output.weight", "offset": 2642841600, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.19.attn_q.weight", "offset": 2654105600, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.19.attn_v.weight", "offset": 2662707200, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 2673971200, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2673991680, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.20.ffn_gate.weight", "offset": 2704404480, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 2734817280, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 2765230080, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 2765250560, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.20.attn_output.weight", "offset": 2773852160, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.20.attn_q.weight", "offset": 2785116160, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.20.attn_v.weight", "offset": 2793717760, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 2804981760, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2805002240, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.21.ffn_gate.weight", "offset": 2835415040, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 2865827840, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 2896240640, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2896261120, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.21.attn_output.weight", "offset": 2904862720, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.21.attn_q.weight", "offset": 2916126720, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.21.attn_v.weight", "offset": 2924728320, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 2935992320, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2936012800, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.22.ffn_gate.weight", "offset": 2966425600, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 2996838400, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 3027251200, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3027271680, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.22.attn_output.weight", "offset": 3035873280, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.22.attn_q.weight", "offset": 3047137280, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.22.attn_v.weight", "offset": 3055738880, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.23.attn_norm.weight", "offset": 3067002880, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3067023360, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.23.ffn_gate.weight", "offset": 3097436160, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 3127848960, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 3158261760, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3158282240, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.23.attn_output.weight", "offset": 3166883840, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.23.attn_q.weight", "offset": 3178147840, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.23.attn_v.weight", "offset": 3186749440, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.24.attn_norm.weight", "offset": 3198013440, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3198033920, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.24.ffn_gate.weight", "offset": 3228446720, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 3258859520, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 3289272320, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 3289292800, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.24.attn_output.weight", "offset": 3297894400, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.24.attn_q.weight", "offset": 3309158400, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.24.attn_v.weight", "offset": 3317760000, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 3329024000, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3329044480, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.25.ffn_gate.weight", "offset": 3359457280, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 3389870080, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 3420282880, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3420303360, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.25.attn_output.weight", "offset": 3428904960, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.25.attn_q.weight", "offset": 3440168960, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.25.attn_v.weight", "offset": 3448770560, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 3460034560, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3460055040, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.26.ffn_gate.weight", "offset": 3490467840, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 3520880640, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 3551293440, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3551313920, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.26.attn_output.weight", "offset": 3559915520, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.26.attn_q.weight", "offset": 3571179520, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.26.attn_v.weight", "offset": 3579781120, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 3591045120, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3591065600, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.27.ffn_gate.weight", "offset": 3621478400, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 3651891200, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 3682304000, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 3682324480, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.27.attn_output.weight", "offset": 3690926080, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.27.attn_q.weight", "offset": 3702190080, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.27.attn_v.weight", "offset": 3710791680, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 3722055680, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3722076160, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.28.ffn_gate.weight", "offset": 3752488960, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 3782901760, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 3813314560, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3813335040, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.28.attn_output.weight", "offset": 3821936640, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.28.attn_q.weight", "offset": 3833200640, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.28.attn_v.weight", "offset": 3841802240, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.29.attn_norm.weight", "offset": 3853066240, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3853086720, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.29.ffn_gate.weight", "offset": 3883499520, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 3913912320, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 3944325120, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 3944345600, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.29.attn_output.weight", "offset": 3952947200, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.29.attn_q.weight", "offset": 3964211200, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.29.attn_v.weight", "offset": 3972812800, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.30.ffn_gate.weight", "offset": 3984076800, "shape": [ 5120, 13824 ], "size": 11264000, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 4014489600, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.30.attn_k.weight", "offset": 4044902400, "shape": [ 5120, 5120 ], "size": 30412800, "type": 10 }, { "name": "blk.30.attn_output.weight", "offset": 4053504000, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.30.attn_q.weight", "offset": 4064768000, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.30.attn_v.weight", "offset": 4073369600, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "output.weight", "offset": 4084633600, "shape": [ 5120, 32000 ], "size": 11264000, "type": 14 }, { "name": "blk.30.attn_norm.weight", "offset": 4219033600, "shape": [ 5120 ], "size": 134400000, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 4219054080, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 4249466880, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 4249487360, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 4249507840, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.31.ffn_gate.weight", "offset": 4279920640, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 4310333440, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 4340746240, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 4340766720, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.31.attn_output.weight", "offset": 4349368320, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.31.attn_q.weight", "offset": 4360632320, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.31.attn_v.weight", "offset": 4369233920, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.32.attn_norm.weight", "offset": 4380497920, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 4380518400, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.32.ffn_gate.weight", "offset": 4410931200, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 4441344000, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 4471756800, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "wvF\ufffde\ufffdM\u0016\ufffd\u0016\u0026'eEf7%3f$", "offset": 4471777280, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.32.attn_output.weight", "offset": 4480378880, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.32.attn_q.weight", "offset": 4491642880, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.32.attn_v.weight", "offset": 4500244480, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.33.attn_norm.weight", "offset": 4511508480, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 4511528960, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.33.ffn_gate.weight", "offset": 4541941760, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 4572354560, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 4602767360, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 4602787840, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.33.attn_output.weight", "offset": 4611389440, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.33.attn_q.weight", "offset": 4622653440, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.33.attn_v.weight", "offset": 4631255040, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.34.attn_norm.weight", "offset": 4642519040, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 4642539520, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.34.ffn_gate.weight", "offset": 4672952320, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 4703365120, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 4733777920, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 4733798400, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.34.attn_output.weight", "offset": 4742400000, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.34.attn_q.weight", "offset": 4753664000, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.34.attn_v.weight", "offset": 4762265600, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.35.attn_norm.weight", "offset": 4773529600, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 4773550080, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.35.ffn_gate.weight", "offset": 4803962880, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 4834375680, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 4864788480, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 4864808960, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.35.attn_output.weight", "offset": 4873410560, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.35.attn_q.weight", "offset": 4884674560, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.35.attn_v.weight", "offset": 4893276160, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.36.attn_norm.weight", "offset": 4904540160, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 4904560640, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.36.ffn_gate.weight", "offset": 4934973440, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 4965386240, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 4995799040, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 4995819520, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.36.attn_output.weight", "offset": 5004421120, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.36.attn_q.weight", "offset": 5015685120, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.36.attn_v.weight", "offset": 5024286720, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.37.attn_norm.weight", "offset": 5035550720, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 5035571200, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.37.ffn_gate.weight", "offset": 5065984000, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 5096396800, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 5126809600, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 5126830080, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.37.attn_output.weight", "offset": 5135431680, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.37.attn_q.weight", "offset": 5146695680, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.37.attn_v.weight", "offset": 5155297280, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.38.attn_norm.weight", "offset": 5166561280, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 5166581760, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.38.ffn_gate.weight", "offset": 5196994560, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 5227407360, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 5257820160, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 5257840640, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.38.attn_output.weight", "offset": 5266442240, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.38.attn_q.weight", "offset": 5277706240, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.38.attn_v.weight", "offset": 5286307840, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.39.attn_norm.weight", "offset": 5297571840, "shape": [ 5120 ], "size": 11264000, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 5297592320, "shape": [ 13824, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.39.ffn_gate.weight", "offset": 5328005120, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 5358417920, "shape": [ 5120, 13824 ], "size": 30412800, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 5388830720, "shape": [ 5120 ], "size": 30412800, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 5388851200, "shape": [ 5120, 5120 ], "size": 20480, "type": 10 }, { "name": "blk.39.attn_output.weight", "offset": 5397452800, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "blk.39.attn_q.weight", "offset": 5408716800, "shape": [ 5120, 5120 ], "size": 11264000, "type": 10 }, { "name": "blk.39.attn_v.weight", "offset": 5417318400, "shape": [ 5120, 5120 ], "size": 8601600, "type": 11 }, { "name": "output_norm.weight", "offset": 5428582400, "shape": [ 5120 ], "size": 11264000, "type": 0 } ], "version": 2 }