taozhiyuai / openbiollm-llama-3

{ "metadata": { "general.architecture": "llama", "general.file_type": 7, "general.name": ".", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 32, "llama.context_length": 8192, "llama.embedding_length": 4096, "llama.feed_forward_length": 14336, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 500000, "llama.vocab_size": 128256, "tokenizer.ggml.bos_token_id": 128000, "tokenizer.ggml.eos_token_id": 128001, "tokenizer.ggml.merges": "... (280147 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 128001, "tokenizer.ggml.scores": "... (128256 values)", "tokenizer.ggml.token_type": "... (128256 values)", "tokenizer.ggml.tokens": "... (128256 values)" }, "num_params": 8030261248, "tensors": [ { "name": "blk.0.attn_q.weight", "offset": 558170112, "shape": [ 4096, 4096 ], "size": 558170112, "type": 8 }, { "name": "blk.0.attn_k.weight", "offset": 575995904, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.0.attn_v.weight", "offset": 580452352, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 584908800, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.0.ffn_gate.weight", "offset": 602734592, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 665124864, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.0.ffn_down.weight", "offset": 727515136, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.0.attn_norm.weight", "offset": 789905408, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 789921792, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 789938176, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.1.attn_k.weight", "offset": 807763968, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.1.attn_v.weight", "offset": 812220416, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 816676864, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.1.ffn_gate.weight", "offset": 834502656, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.1.ffn_up.weight", "offset": 896892928, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.1.ffn_down.weight", "offset": 959283200, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 1021673472, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 1021689856, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 1021706240, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.2.attn_k.weight", "offset": 1039532032, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.2.attn_v.weight", "offset": 1043988480, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 1048444928, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.2.ffn_gate.weight", "offset": 1066270720, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.2.ffn_up.weight", "offset": 1128660992, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.2.ffn_down.weight", "offset": 1191051264, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 1253441536, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 1253457920, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 1253474304, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.3.attn_k.weight", "offset": 1271300096, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.3.attn_v.weight", "offset": 1275756544, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 1280212992, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.3.ffn_gate.weight", "offset": 1298038784, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.3.ffn_up.weight", "offset": 1360429056, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.3.ffn_down.weight", "offset": 1422819328, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.3.attn_norm.weight", "offset": 1485209600, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 1485225984, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 1485242368, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.4.attn_k.weight", "offset": 1503068160, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.4.attn_v.weight", "offset": 1507524608, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 1511981056, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.4.ffn_gate.weight", "offset": 1529806848, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.4.ffn_up.weight", "offset": 1592197120, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.4.ffn_down.weight", "offset": 1654587392, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 1716977664, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 1716994048, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1717010432, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.5.attn_k.weight", "offset": 1734836224, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.5.attn_v.weight", "offset": 1739292672, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 1743749120, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.5.ffn_gate.weight", "offset": 1761574912, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.5.ffn_up.weight", "offset": 1823965184, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.5.ffn_down.weight", "offset": 1886355456, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.5.attn_norm.weight", "offset": 1948745728, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 1948762112, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1948778496, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.6.attn_k.weight", "offset": 1966604288, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.6.attn_v.weight", "offset": 1971060736, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 1975517184, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.6.ffn_gate.weight", "offset": 1993342976, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.6.ffn_up.weight", "offset": 2055733248, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.6.ffn_down.weight", "offset": 2118123520, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.6.attn_norm.weight", "offset": 2180513792, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 2180530176, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 2180546560, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.7.attn_k.weight", "offset": 2198372352, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.7.attn_v.weight", "offset": 2202828800, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 2207285248, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.7.ffn_gate.weight", "offset": 2225111040, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 2287501312, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.7.ffn_down.weight", "offset": 2349891584, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.7.attn_norm.weight", "offset": 2412281856, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 2412298240, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 2412314624, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.8.attn_k.weight", "offset": 2430140416, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.8.attn_v.weight", "offset": 2434596864, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 2439053312, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.8.ffn_gate.weight", "offset": 2456879104, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.8.ffn_up.weight", "offset": 2519269376, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.8.ffn_down.weight", "offset": 2581659648, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.8.attn_norm.weight", "offset": 2644049920, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 2644066304, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 2644082688, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.9.attn_k.weight", "offset": 2661908480, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.9.attn_v.weight", "offset": 2666364928, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 2670821376, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.9.ffn_gate.weight", "offset": 2688647168, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.9.ffn_up.weight", "offset": 2751037440, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.9.ffn_down.weight", "offset": 2813427712, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 2875817984, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 2875834368, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 2875850752, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.10.attn_k.weight", "offset": 2893676544, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.10.attn_v.weight", "offset": 2898132992, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 2902589440, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.10.ffn_gate.weight", "offset": 2920415232, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.10.ffn_up.weight", "offset": 2982805504, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.10.ffn_down.weight", "offset": 3045195776, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 3107586048, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 3107602432, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 3107618816, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.11.attn_k.weight", "offset": 3125444608, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.11.attn_v.weight", "offset": 3129901056, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 3134357504, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.11.ffn_gate.weight", "offset": 3152183296, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.11.ffn_up.weight", "offset": 3214573568, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.11.ffn_down.weight", "offset": 3276963840, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 3339354112, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 3339370496, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 3339386880, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.12.attn_k.weight", "offset": 3357212672, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.12.attn_v.weight", "offset": 3361669120, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 3366125568, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.12.ffn_gate.weight", "offset": 3383951360, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.12.ffn_up.weight", "offset": 3446341632, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.12.ffn_down.weight", "offset": 3508731904, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.12.attn_norm.weight", "offset": 3571122176, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 3571138560, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 3571154944, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.13.attn_k.weight", "offset": 3588980736, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.13.attn_v.weight", "offset": 3593437184, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 3597893632, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.13.ffn_gate.weight", "offset": 3615719424, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.13.ffn_up.weight", "offset": 3678109696, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.13.ffn_down.weight", "offset": 3740499968, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.13.attn_norm.weight", "offset": 3802890240, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 3802906624, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 3802923008, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.14.attn_k.weight", "offset": 3820748800, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.14.attn_v.weight", "offset": 3825205248, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 3829661696, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.14.ffn_gate.weight", "offset": 3847487488, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.14.ffn_up.weight", "offset": 3909877760, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.14.ffn_down.weight", "offset": 3972268032, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 4034658304, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 4034674688, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 4034691072, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.15.attn_k.weight", "offset": 4052516864, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000blk.30.ffn_gat", "offset": 4056973312, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 4061429760, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.15.ffn_gate.weight", "offset": 4079255552, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.15.ffn_up.weight", "offset": 4141645824, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.15.ffn_down.weight", "offset": 4204036096, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.15.attn_norm.weight", "offset": 4266426368, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 4266442752, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 4266459136, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.16.attn_k.weight", "offset": 4284284928, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.16.attn_v.weight", "offset": 4288741376, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 4293197824, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.16.ffn_gate.weight", "offset": 4311023616, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.16.ffn_up.weight", "offset": 4373413888, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.16.ffn_down.weight", "offset": 4435804160, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.16.attn_norm.weight", "offset": 4498194432, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 4498210816, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 4498227200, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.17.attn_k.weight", "offset": 4516052992, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.17.attn_v.weight", "offset": 4520509440, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 4524965888, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.17.ffn_gate.weight", "offset": 4542791680, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.17.ffn_up.weight", "offset": 4605181952, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.17.ffn_down.weight", "offset": 4667572224, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 4729962496, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 4729978880, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 4729995264, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.18.attn_k.weight", "offset": 4747821056, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.18.attn_v.weight", "offset": 4752277504, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 4756733952, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.18.ffn_gate.weight", "offset": 4774559744, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.18.ffn_up.weight", "offset": 4836950016, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.18.ffn_down.weight", "offset": 4899340288, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.18.attn_norm.weight", "offset": 4961730560, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 4961746944, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 4961763328, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.19.attn_k.weight", "offset": 4979589120, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.19.attn_v.weight", "offset": 4984045568, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 4988502016, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.19.ffn_gate.weight", "offset": 5006327808, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.19.ffn_up.weight", "offset": 5068718080, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.19.ffn_down.weight", "offset": 5131108352, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.19.attn_norm.weight", "offset": 5193498624, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 5193515008, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 5193531392, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.20.attn_k.weight", "offset": 5211357184, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.20.attn_v.weight", "offset": 5215813632, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 5220270080, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.20.ffn_gate.weight", "offset": 5238095872, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.20.ffn_up.weight", "offset": 5300486144, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.20.ffn_down.weight", "offset": 5362876416, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.20.attn_norm.weight", "offset": 5425266688, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 5425283072, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 5425299456, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.21.attn_k.weight", "offset": 5443125248, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.21.attn_v.weight", "offset": 5447581696, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 5452038144, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.21.ffn_gate.weight", "offset": 5469863936, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.21.ffn_up.weight", "offset": 5532254208, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.21.ffn_down.weight", "offset": 5594644480, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.21.attn_norm.weight", "offset": 5657034752, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 5657051136, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 5657067520, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.22.attn_k.weight", "offset": 5674893312, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.22.attn_v.weight", "offset": 5679349760, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 5683806208, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.22.ffn_gate.weight", "offset": 5701632000, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.22.ffn_up.weight", "offset": 5764022272, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.22.ffn_down.weight", "offset": 5826412544, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 5888802816, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 5888819200, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 5888835584, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.23.attn_k.weight", "offset": 5906661376, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.23.attn_v.weight", "offset": 5911117824, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 5915574272, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.23.ffn_gate.weight", "offset": 5933400064, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.23.ffn_up.weight", "offset": 5995790336, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.23.ffn_down.weight", "offset": 6058180608, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.23.attn_norm.weight", "offset": 6120570880, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 6120587264, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 6120603648, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.24.attn_k.weight", "offset": 6138429440, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.24.attn_v.weight", "offset": 6142885888, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 6147342336, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.24.ffn_gate.weight", "offset": 6165168128, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.24.ffn_up.weight", "offset": 6227558400, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.24.ffn_down.weight", "offset": 6289948672, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.24.attn_norm.weight", "offset": 6352338944, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 6352355328, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 6352371712, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.25.attn_k.weight", "offset": 6370197504, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.25.attn_v.weight", "offset": 6374653952, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 6379110400, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.25.ffn_gate.weight", "offset": 6396936192, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.25.ffn_up.weight", "offset": 6459326464, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.25.ffn_down.weight", "offset": 6521716736, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.25.attn_norm.weight", "offset": 6584107008, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 6584123392, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 6584139776, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.26.attn_k.weight", "offset": 6601965568, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.26.attn_v.weight", "offset": 6606422016, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 6610878464, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.26.ffn_gate.weight", "offset": 6628704256, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.26.ffn_up.weight", "offset": 6691094528, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.26.ffn_down.weight", "offset": 6753484800, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.26.attn_norm.weight", "offset": 6815875072, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 6815891456, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 6815907840, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.27.attn_k.weight", "offset": 6833733632, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.27.attn_v.weight", "offset": 6838190080, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 6842646528, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.27.ffn_gate.weight", "offset": 6860472320, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.27.ffn_up.weight", "offset": 6922862592, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.27.ffn_down.weight", "offset": 6985252864, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.27.attn_norm.weight", "offset": 7047643136, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 7047659520, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 7047675904, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.28.attn_k.weight", "offset": 7065501696, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.28.attn_v.weight", "offset": 7069958144, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 7074414592, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.28.ffn_gate.weight", "offset": 7092240384, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.28.ffn_up.weight", "offset": 7154630656, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.28.ffn_down.weight", "offset": 7217020928, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.28.attn_norm.weight", "offset": 7279411200, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 7279427584, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 7279443968, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.29.attn_k.weight", "offset": 7297269760, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.29.attn_v.weight", "offset": 7301726208, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 7306182656, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.29.ffn_gate.weight", "offset": 7324008448, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.29.ffn_up.weight", "offset": 7386398720, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.29.ffn_down.weight", "offset": 7448788992, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.29.attn_norm.weight", "offset": 7511179264, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 7511195648, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 7511212032, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.30.attn_k.weight", "offset": 7529037824, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.30.attn_v.weight", "offset": 7533494272, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 7537950720, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "\ufffd\ufffd\ufffd'\ufffd\ufffd\ufffdѡ#\ufffd\ufffd\ufffd+\ufffd\ufffd\u0001\u00110\ufffd\ufffd", "offset": 7555776512, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.30.ffn_up.weight", "offset": 7618166784, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.30.ffn_down.weight", "offset": 7680557056, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.30.attn_norm.weight", "offset": 7742947328, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 7742963712, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 7742980096, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.31.attn_k.weight", "offset": 7760805888, "shape": [ 4096, 1024 ], "size": 17825792, "type": 8 }, { "name": "blk.31.attn_v.weight", "offset": 7765262336, "shape": [ 4096, 1024 ], "size": 4456448, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 7769718784, "shape": [ 4096, 4096 ], "size": 4456448, "type": 8 }, { "name": "blk.31.ffn_gate.weight", "offset": 7787544576, "shape": [ 4096, 14336 ], "size": 17825792, "type": 8 }, { "name": "blk.31.ffn_up.weight", "offset": 7849934848, "shape": [ 4096, 14336 ], "size": 62390272, "type": 8 }, { "name": "blk.31.ffn_down.weight", "offset": 7912325120, "shape": [ 14336, 4096 ], "size": 62390272, "type": 8 }, { "name": "blk.31.attn_norm.weight", "offset": 7974715392, "shape": [ 4096 ], "size": 62390272, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 7974731776, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "output_norm.weight", "offset": 7974748160, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "output.weight", "offset": 7974764544, "shape": [ 4096, 128256 ], "size": 16384, "type": 8 } ], "version": 3 }