codellama

{ "metadata": { "general.architecture": "llama", "general.file_type": 14, "general.name": "codellama", "general.quantization_version": 2, "llama.attention.head_count": 40, "llama.attention.head_count_kv": 40, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 40, "llama.context_length": 16384, "llama.embedding_length": 5120, "llama.feed_forward_length": 13824, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32000 values)", "tokenizer.ggml.token_type": "... (32000 values)", "tokenizer.ggml.tokens": "... (32000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13015864320, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 92160000, "shape": [ 5120 ], "size": 92160000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 92180480, "shape": [ 13824, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 140840960, "shape": [ 5120, 13824 ], "size": 48660480, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 180654080, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.0.ffn_norm.weight", "offset": 220467200, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 220487680, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 235233280, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 249978880, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.0.attn_v.weight", "offset": 264724480, "shape": [ 5120, 5120 ], "size": 14745600, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 282746880, "shape": [ 5120 ], "size": 18022400, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 282767360, "shape": [ 13824, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 331427840, "shape": [ 5120, 13824 ], "size": 48660480, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 371240960, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 411054080, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 411074560, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 425820160, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 440565760, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.1.attn_v.weight", "offset": 455311360, "shape": [ 5120, 5120 ], "size": 14745600, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 473333760, "shape": [ 5120 ], "size": 18022400, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 473354240, "shape": [ 13824, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 522014720, "shape": [ 5120, 13824 ], "size": 48660480, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 561827840, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 601640960, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 601661440, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 616407040, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.10.attn_q.weight", "offset": 631152640, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.10.attn_v.weight", "offset": 645898240, "shape": [ 5120, 5120 ], "size": 14745600, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 663920640, "shape": [ 5120 ], "size": 18022400, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 663941120, "shape": [ 13824, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 712601600, "shape": [ 5120, 13824 ], "size": 48660480, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 752414720, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 792227840, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 792248320, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 806993920, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.11.attn_q.weight", "offset": 821739520, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.11.attn_v.weight", "offset": 836485120, "shape": [ 5120, 5120 ], "size": 14745600, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 854507520, "shape": [ 5120 ], "size": 18022400, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 854528000, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 894341120, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 934154240, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 973967360, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 973987840, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 988733440, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.12.attn_q.weight", "offset": 1003479040, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.12.attn_v.weight", "offset": 1018224640, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 1032970240, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1032990720, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 1072803840, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 1112616960, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 1152430080, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1152450560, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 1167196160, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.13.attn_q.weight", "offset": 1181941760, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.13.attn_v.weight", "offset": 1196687360, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 1211432960, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1211453440, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 1251266560, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 1291079680, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 1330892800, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1330913280, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 1345658880, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 1360404480, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.14.attn_v.weight", "offset": 1375150080, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.15.attn_k.weight", "offset": 1389895680, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 1404641280, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 1419386880, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1419407360, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.2.ffn_gate.weight", "offset": 1459220480, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 1499033600, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 1538846720, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1538867200, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 1553612800, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 1568358400, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.2.attn_v.weight", "offset": 1583104000, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 1597849600, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1597870080, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 1637683200, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 1677496320, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 1717309440, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1717329920, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 1732075520, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 1746821120, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.3.attn_v.weight", "offset": 1761566720, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 1776312320, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1776332800, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 1816145920, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 1855959040, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 1895772160, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1895792640, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 1910538240, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 1925283840, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.4.attn_v.weight", "offset": 1940029440, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 1954775040, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1954795520, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 1994608640, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 2034421760, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 2074234880, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 2074255360, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 2089000960, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 2103746560, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.5.attn_v.weight", "offset": 2118492160, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 2133237760, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2133258240, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 2173071360, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 2212884480, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 2252697600, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 2252718080, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 2267463680, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 2282209280, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.6.attn_v.weight", "offset": 2296954880, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 2311700480, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2311720960, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 2351534080, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 2391347200, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 2431160320, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2431180800, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 2445926400, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 2460672000, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.7.attn_v.weight", "offset": 2475417600, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 2490163200, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2490183680, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 2529996800, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 2569809920, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 2609623040, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2609643520, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 2624389120, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 2639134720, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.8.attn_v.weight", "offset": 2653880320, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 2668625920, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2668646400, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 2708459520, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 2748272640, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.9.ffn_norm.weight", "offset": 2788085760, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2788106240, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 2802851840, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 2817597440, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.9.attn_v.weight", "offset": 2832343040, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 2847088640, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2847109120, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 2886922240, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 2926735360, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 2966548480, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 2966568960, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.attn_v.weight", "offset": 2981314560, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 2996060160, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2996080640, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 3035893760, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 3075706880, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 3115520000, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 3115540480, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 3130286080, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.16.attn_q.weight", "offset": 3145031680, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.16.attn_v.weight", "offset": 3159777280, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 3174522880, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3174543360, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 3214356480, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 3254169600, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "\u0000\u0000\u0000\u0000\u00000V\ufffd\u0000\u0000\u0000\u0000\u0014\u0000\u0000\u0000\u0000\u0000\u0000\u0000bl", "offset": 3293982720, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 3294003200, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 3308748800, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 3323494400, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.17.attn_v.weight", "offset": 3338240000, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 3352985600, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3353006080, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 3392819200, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 3432632320, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 3472445440, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 3472465920, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 3487211520, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.18.attn_q.weight", "offset": 3501957120, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.18.attn_v.weight", "offset": 3516702720, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 3531448320, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3531468800, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 3571281920, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 3611095040, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.19.ffn_norm.weight", "offset": 3650908160, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 3650928640, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 3665674240, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 3680419840, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.19.attn_v.weight", "offset": 3695165440, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 3709911040, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3709931520, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 3749744640, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 3789557760, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 3829370880, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 3829391360, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 3844136960, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 3858882560, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.20.attn_v.weight", "offset": 3873628160, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 3888373760, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3888394240, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 3928207360, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 3968020480, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 4007833600, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 4007854080, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 4022599680, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 4037345280, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.21.attn_v.weight", "offset": 4052090880, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 4066836480, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4066856960, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 4106670080, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 4146483200, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 4186296320, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 4186316800, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 4201062400, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 4215808000, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.22.attn_v.weight", "offset": 4230553600, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 4245299200, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4245319680, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 4285132800, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 4324945920, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 4364759040, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 4364779520, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 4379525120, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 4394270720, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.23.attn_v.weight", "offset": 4409016320, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 4423761920, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4423782400, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 4463595520, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 4503408640, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 4543221760, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 4543242240, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 4557987840, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 4572733440, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.24.attn_v.weight", "offset": 4587479040, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 4602224640, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4602245120, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 4642058240, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 4681871360, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 4721684480, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 4721704960, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 4736450560, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 4751196160, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.25.attn_v.weight", "offset": 4765941760, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 4780687360, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4780707840, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 4820520960, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 4860334080, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 4900147200, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 4900167680, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 4914913280, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 4929658880, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.26.attn_v.weight", "offset": 4944404480, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 4959150080, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4959170560, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 4998983680, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 5038796800, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.27.ffn_norm.weight", "offset": 5078609920, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 5078630400, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 5093376000, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.27.attn_q.weight", "offset": 5108121600, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.27.attn_v.weight", "offset": 5122867200, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 5137612800, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5137633280, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.28.ffn_gate.weight", "offset": 5177446400, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 5217259520, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.28.ffn_norm.weight", "offset": 5257072640, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 5257093120, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 5271838720, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.28.attn_q.weight", "offset": 5286584320, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.28.attn_v.weight", "offset": 5301329920, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 5316075520, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 5316096000, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.ffn_gate.weight", "offset": 5355909120, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 5395722240, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.29.ffn_norm.weight", "offset": 5435535360, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 5435555840, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 5450301440, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.29.attn_q.weight", "offset": 5465047040, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.29.attn_v.weight", "offset": 5479792640, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.30.ffn_gate.weight", "offset": 5494538240, "shape": [ 5120, 13824 ], "size": 14745600, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 5534351360, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.30.attn_k.weight", "offset": 5574164480, "shape": [ 5120, 5120 ], "size": 39813120, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 5588910080, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.30.attn_q.weight", "offset": 5603655680, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.30.attn_v.weight", "offset": 5618401280, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "output.weight", "offset": 5633146880, "shape": [ 5120, 32000 ], "size": 14745600, "type": 14 }, { "name": "blk.30.attn_norm.weight", "offset": 5767546880, "shape": [ 5120 ], "size": 134400000, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 5767567360, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.30.ffn_norm.weight", "offset": 5807380480, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 5807400960, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 5807421440, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.31.ffn_gate.weight", "offset": 5847234560, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 5887047680, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.31.ffn_norm.weight", "offset": 5926860800, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 5926881280, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.31.attn_output.weight", "offset": 5941626880, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.31.attn_q.weight", "offset": 5956372480, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.31.attn_v.weight", "offset": 5971118080, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 5985863680, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 5985884160, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.ffn_gate.weight", "offset": 6025697280, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 6065510400, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.32.ffn_norm.weight", "offset": 6105323520, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "\ufffd\ufffd\ufffd\nʩ\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd", "offset": 6105344000, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.attn_output.weight", "offset": 6120089600, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.32.attn_q.weight", "offset": 6134835200, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.32.attn_v.weight", "offset": 6149580800, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 6164326400, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6164346880, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.ffn_gate.weight", "offset": 6204160000, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 6243973120, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.33.ffn_norm.weight", "offset": 6283786240, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 6283806720, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.attn_output.weight", "offset": 6298552320, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.33.attn_q.weight", "offset": 6313297920, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.33.attn_v.weight", "offset": 6328043520, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 6342789120, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 6342809600, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.34.ffn_gate.weight", "offset": 6382622720, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.34.ffn_up.weight", "offset": 6422435840, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.34.ffn_norm.weight", "offset": 6462248960, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 6462269440, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.34.attn_output.weight", "offset": 6477015040, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.34.attn_q.weight", "offset": 6491760640, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.34.attn_v.weight", "offset": 6506506240, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.35.attn_norm.weight", "offset": 6521251840, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 6521272320, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.35.ffn_gate.weight", "offset": 6561085440, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 6600898560, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.35.ffn_norm.weight", "offset": 6640711680, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 6640732160, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 6655477760, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.35.attn_q.weight", "offset": 6670223360, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.35.attn_v.weight", "offset": 6684968960, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.36.attn_norm.weight", "offset": 6699714560, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 6699735040, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.36.ffn_gate.weight", "offset": 6739548160, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 6779361280, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.36.ffn_norm.weight", "offset": 6819174400, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 6819194880, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.36.attn_output.weight", "offset": 6833940480, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.36.attn_q.weight", "offset": 6848686080, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.36.attn_v.weight", "offset": 6863431680, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 6878177280, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 6878197760, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.37.ffn_gate.weight", "offset": 6918010880, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 6957824000, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.37.ffn_norm.weight", "offset": 6997637120, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 6997657600, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.37.attn_output.weight", "offset": 7012403200, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.37.attn_q.weight", "offset": 7027148800, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.37.attn_v.weight", "offset": 7041894400, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 7056640000, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 7056660480, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.38.ffn_gate.weight", "offset": 7096473600, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 7136286720, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.38.ffn_norm.weight", "offset": 7176099840, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 7176120320, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.38.attn_output.weight", "offset": 7190865920, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.38.attn_q.weight", "offset": 7205611520, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.38.attn_v.weight", "offset": 7220357120, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 7235102720, "shape": [ 5120 ], "size": 14745600, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 7235123200, "shape": [ 13824, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.39.ffn_gate.weight", "offset": 7274936320, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 7314749440, "shape": [ 5120, 13824 ], "size": 39813120, "type": 12 }, { "name": "blk.39.ffn_norm.weight", "offset": 7354562560, "shape": [ 5120 ], "size": 39813120, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 7354583040, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.39.attn_output.weight", "offset": 7369328640, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.39.attn_q.weight", "offset": 7384074240, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "blk.39.attn_v.weight", "offset": 7398819840, "shape": [ 5120, 5120 ], "size": 14745600, "type": 12 }, { "name": "output_norm.weight", "offset": 7413565440, "shape": [ 5120 ], "size": 14745600, "type": 0 } ], "version": 2 }