wizard-vicuna-uncensored

{ "metadata": { "general.architecture": "llama", "general.file_type": 18, "general.name": "ehartford", "general.quantization_version": 2, "llama.attention.head_count": 40, "llama.attention.head_count_kv": 40, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 40, "llama.context_length": 2048, "llama.embedding_length": 5120, "llama.feed_forward_length": 13824, "llama.rope.dimension_count": 128, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32000 values)", "tokenizer.ggml.token_type": "... (32000 values)", "tokenizer.ggml.tokens": "... (32000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13015864320, "tensors": [ { "name": "blk.0.attn_q.weight", "offset": 134400000, "shape": [ 5120, 5120 ], "size": 134400000, "type": 14 }, { "name": "blk.0.attn_k.weight", "offset": 155904000, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.0.attn_v.weight", "offset": 177408000, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.0.attn_output.weight", "offset": 198912000, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 220416000, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.0.ffn_down.weight", "offset": 278476800, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 336537600, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.0.attn_norm.weight", "offset": 394598400, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 394618880, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 394639360, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.1.attn_k.weight", "offset": 416143360, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.1.attn_v.weight", "offset": 437647360, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.1.attn_output.weight", "offset": 459151360, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 480655360, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.1.ffn_down.weight", "offset": 538716160, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 596776960, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 654837760, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 654858240, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 654878720, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.2.attn_k.weight", "offset": 676382720, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.2.attn_v.weight", "offset": 697886720, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.2.attn_output.weight", "offset": 719390720, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 740894720, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.2.ffn_down.weight", "offset": 798955520, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 857016320, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 915077120, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 915097600, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 915118080, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.3.attn_k.weight", "offset": 936622080, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.3.attn_v.weight", "offset": 958126080, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.3.attn_output.weight", "offset": 979630080, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 1001134080, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.3.ffn_down.weight", "offset": 1059194880, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.3.ffn_up.weight", "offset": 1117255680, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 1175316480, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 1175336960, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 1175357440, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.4.attn_k.weight", "offset": 1196861440, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.4.attn_v.weight", "offset": 1218365440, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.4.attn_output.weight", "offset": 1239869440, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 1261373440, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.4.ffn_down.weight", "offset": 1319434240, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 1377495040, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1435555840, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 1435576320, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1435596800, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.5.attn_k.weight", "offset": 1457100800, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.5.attn_v.weight", "offset": 1478604800, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.5.attn_output.weight", "offset": 1500108800, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.5.ffn_gate.weight", "offset": 1521612800, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.5.ffn_down.weight", "offset": 1579673600, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.5.ffn_up.weight", "offset": 1637734400, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 1695795200, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 1695815680, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1695836160, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.6.attn_k.weight", "offset": 1717340160, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.6.attn_v.weight", "offset": 1738844160, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.6.attn_output.weight", "offset": 1760348160, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.6.ffn_gate.weight", "offset": 1781852160, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.6.ffn_down.weight", "offset": 1839912960, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.6.ffn_up.weight", "offset": 1897973760, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 1956034560, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 1956055040, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 1956075520, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.7.attn_k.weight", "offset": 1977579520, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.7.attn_v.weight", "offset": 1999083520, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.7.attn_output.weight", "offset": 2020587520, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 2042091520, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.7.ffn_down.weight", "offset": 2100152320, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.7.ffn_up.weight", "offset": 2158213120, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 2216273920, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 2216294400, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 2216314880, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.8.attn_k.weight", "offset": 2237818880, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.8.attn_v.weight", "offset": 2259322880, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.8.attn_output.weight", "offset": 2280826880, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.8.ffn_gate.weight", "offset": 2302330880, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.8.ffn_down.weight", "offset": 2360391680, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.8.ffn_up.weight", "offset": 2418452480, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.8.attn_norm.weight", "offset": 2476513280, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 2476533760, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 2476554240, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.9.attn_k.weight", "offset": 2498058240, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.9.attn_v.weight", "offset": 2519562240, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.9.attn_output.weight", "offset": 2541066240, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 2562570240, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.9.ffn_down.weight", "offset": 2620631040, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.9.ffn_up.weight", "offset": 2678691840, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 2736752640, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 2736773120, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 2736793600, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.10.attn_k.weight", "offset": 2758297600, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.10.attn_v.weight", "offset": 2779801600, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.10.attn_output.weight", "offset": 2801305600, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 2822809600, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.10.ffn_down.weight", "offset": 2880870400, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.10.ffn_up.weight", "offset": 2938931200, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 2996992000, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 2997012480, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 2997032960, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.11.attn_k.weight", "offset": 3018536960, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.11.attn_v.weight", "offset": 3040040960, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.11.attn_output.weight", "offset": 3061544960, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.11.ffn_gate.weight", "offset": 3083048960, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.11.ffn_down.weight", "offset": 3141109760, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.11.ffn_up.weight", "offset": 3199170560, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 3257231360, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 3257251840, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 3257272320, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.12.attn_k.weight", "offset": 3278776320, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.12.attn_v.weight", "offset": 3300280320, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.12.attn_output.weight", "offset": 3321784320, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.12.ffn_gate.weight", "offset": 3343288320, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.12.ffn_down.weight", "offset": 3401349120, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.12.ffn_up.weight", "offset": 3459409920, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 3517470720, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 3517491200, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 3517511680, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.13.attn_k.weight", "offset": 3539015680, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.13.attn_v.weight", "offset": 3560519680, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.13.attn_output.weight", "offset": 3582023680, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.13.ffn_gate.weight", "offset": 3603527680, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.13.ffn_down.weight", "offset": 3661588480, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.13.ffn_up.weight", "offset": 3719649280, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 3777710080, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 3777730560, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 3777751040, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.14.attn_k.weight", "offset": 3799255040, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.14.attn_v.weight", "offset": 3820759040, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.14.attn_output.weight", "offset": 3842263040, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.14.ffn_gate.weight", "offset": 3863767040, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.14.ffn_down.weight", "offset": 3921827840, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.14.ffn_up.weight", "offset": 3979888640, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 4037949440, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 4037969920, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 4037990400, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.15.attn_k.weight", "offset": 4059494400, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.15.attn_v.weight", "offset": 4080998400, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.15.attn_output.weight", "offset": 4102502400, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.15.ffn_gate.weight", "offset": 4124006400, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.15.ffn_down.weight", "offset": 4182067200, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.15.ffn_up.weight", "offset": 4240128000, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.15.attn_norm.weight", "offset": 4298188800, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 4298209280, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 4298229760, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.16.attn_k.weight", "offset": 4319733760, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.16.attn_v.weight", "offset": 4341237760, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.16.attn_output.weight", "offset": 4362741760, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 4384245760, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.16.ffn_down.weight", "offset": 4442306560, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.16.ffn_up.weight", "offset": 4500367360, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 4558428160, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 4558448640, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 4558469120, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.17.attn_k.weight", "offset": 4579973120, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.17.attn_v.weight", "offset": 4601477120, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.17.attn_output.weight", "offset": 4622981120, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.17.ffn_gate.weight", "offset": 4644485120, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.17.ffn_down.weight", "offset": 4702545920, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.17.ffn_up.weight", "offset": 4760606720, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 4818667520, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 4818688000, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 4818708480, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.18.attn_k.weight", "offset": 4840212480, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.18.attn_v.weight", "offset": 4861716480, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.18.attn_output.weight", "offset": 4883220480, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.18.ffn_gate.weight", "offset": 4904724480, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.18.ffn_down.weight", "offset": 4962785280, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.18.ffn_up.weight", "offset": 5020846080, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 5078906880, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 5078927360, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 5078947840, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.19.attn_k.weight", "offset": 5100451840, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.19.attn_v.weight", "offset": 5121955840, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.19.attn_output.weight", "offset": 5143459840, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.19.ffn_gate.weight", "offset": 5164963840, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.19.ffn_down.weight", "offset": 5223024640, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 5281085440, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 5339146240, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 5339166720, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 5339187200, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.20.attn_k.weight", "offset": 5360691200, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.20.attn_v.weight", "offset": 5382195200, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.20.attn_output.weight", "offset": 5403699200, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.20.ffn_gate.weight", "offset": 5425203200, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.20.ffn_down.weight", "offset": 5483264000, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.20.ffn_up.weight", "offset": 5541324800, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.20.attn_norm.weight", "offset": 5599385600, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 5599406080, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 5599426560, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.21.attn_k.weight", "offset": 5620930560, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.21.attn_v.weight", "offset": 5642434560, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.21.attn_output.weight", "offset": 5663938560, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.21.ffn_gate.weight", "offset": 5685442560, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.21.ffn_down.weight", "offset": 5743503360, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.21.ffn_up.weight", "offset": 5801564160, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 5859624960, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 5859645440, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 5859665920, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.22.attn_k.weight", "offset": 5881169920, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.22.attn_v.weight", "offset": 5902673920, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.22.attn_output.weight", "offset": 5924177920, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.22.ffn_gate.weight", "offset": 5945681920, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.22.ffn_down.weight", "offset": 6003742720, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.22.ffn_up.weight", "offset": 6061803520, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 6119864320, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 6119884800, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 6119905280, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.23.attn_k.weight", "offset": 6141409280, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.23.attn_v.weight", "offset": 6162913280, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.23.attn_output.weight", "offset": 6184417280, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.23.ffn_gate.weight", "offset": 6205921280, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.23.ffn_down.weight", "offset": 6263982080, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.23.ffn_up.weight", "offset": 6322042880, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.23.attn_norm.weight", "offset": 6380103680, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 6380124160, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 6380144640, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.24.attn_k.weight", "offset": 6401648640, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.24.attn_v.weight", "offset": 6423152640, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.24.attn_output.weight", "offset": 6444656640, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.24.ffn_gate.weight", "offset": 6466160640, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.24.ffn_down.weight", "offset": 6524221440, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.24.ffn_up.weight", "offset": 6582282240, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 6640343040, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 6640363520, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 6640384000, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.25.attn_k.weight", "offset": 6661888000, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.25.attn_v.weight", "offset": 6683392000, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.25.attn_output.weight", "offset": 6704896000, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 6726400000, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.25.ffn_down.weight", "offset": 6784460800, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.25.ffn_up.weight", "offset": 6842521600, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 6900582400, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 6900602880, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 6900623360, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.26.attn_k.weight", "offset": 6922127360, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.26.attn_v.weight", "offset": 6943631360, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.26.attn_output.weight", "offset": 6965135360, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.26.ffn_gate.weight", "offset": 6986639360, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.26.ffn_down.weight", "offset": 7044700160, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.26.ffn_up.weight", "offset": 7102760960, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 7160821760, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 7160842240, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 7160862720, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.27.attn_k.weight", "offset": 7182366720, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.27.attn_v.weight", "offset": 7203870720, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.27.attn_output.weight", "offset": 7225374720, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 7246878720, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.27.ffn_down.weight", "offset": 7304939520, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.27.ffn_up.weight", "offset": 7363000320, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 7421061120, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 7421081600, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 7421102080, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.28.attn_k.weight", "offset": 7442606080, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.28.attn_v.weight", "offset": 7464110080, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.28.attn_output.weight", "offset": 7485614080, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.28.ffn_gate.weight", "offset": 7507118080, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.28.ffn_down.weight", "offset": 7565178880, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.28.ffn_up.weight", "offset": 7623239680, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.28.attn_norm.weight", "offset": 7681300480, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 7681320960, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 7681341440, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.29.attn_k.weight", "offset": 7702845440, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.29.attn_v.weight", "offset": 7724349440, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.29.attn_output.weight", "offset": 7745853440, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.29.ffn_gate.weight", "offset": 7767357440, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.29.ffn_down.weight", "offset": 7825418240, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.29.ffn_up.weight", "offset": 7883479040, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.29.attn_norm.weight", "offset": 7941539840, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 7941560320, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 7941580800, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.30.attn_k.weight", "offset": 7963084800, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.30.attn_v.weight", "offset": 7984588800, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.30.attn_output.weight", "offset": 8006092800, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.30.ffn_gate.weight", "offset": 8027596800, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.30.ffn_down.weight", "offset": 8085657600, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.30.ffn_up.weight", "offset": 8143718400, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.30.attn_norm.weight", "offset": 8201779200, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 8201799680, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 8201820160, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.31.attn_k.weight", "offset": 8223324160, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.31.attn_v.weight", "offset": 8244828160, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.31.attn_output.weight", "offset": 8266332160, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.31.ffn_gate.weight", "offset": 8287836160, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.31.ffn_down.weight", "offset": 8345896960, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.31.ffn_up.weight", "offset": 8403957760, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.31.attn_norm.weight", "offset": 8462018560, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 8462039040, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.32.attn_q.weight", "offset": 8462059520, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.32.attn_k.weight", "offset": 8483563520, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.32.attn_v.weight", "offset": 8505067520, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.32.attn_output.weight", "offset": 8526571520, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.32.ffn_gate.weight", "offset": 8548075520, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.32.ffn_down.weight", "offset": 8606136320, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.32.ffn_up.weight", "offset": 8664197120, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.32.attn_norm.weight", "offset": 8722257920, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 8722278400, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.33.attn_q.weight", "offset": 8722298880, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.33.attn_k.weight", "offset": 8743802880, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.33.attn_v.weight", "offset": 8765306880, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.33.attn_output.weight", "offset": 8786810880, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.33.ffn_gate.weight", "offset": 8808314880, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.33.ffn_down.weight", "offset": 8866375680, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.33.ffn_up.weight", "offset": 8924436480, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.33.attn_norm.weight", "offset": 8982497280, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 8982517760, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.34.attn_q.weight", "offset": 8982538240, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.34.attn_k.weight", "offset": 9004042240, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.34.attn_v.weight", "offset": 9025546240, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.34.attn_output.weight", "offset": 9047050240, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.34.ffn_gate.weight", "offset": 9068554240, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.34.ffn_down.weight", "offset": 9126615040, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.34.ffn_up.weight", "offset": 9184675840, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.34.attn_norm.weight", "offset": 9242736640, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 9242757120, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.35.attn_q.weight", "offset": 9242777600, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.35.attn_k.weight", "offset": 9264281600, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.35.attn_v.weight", "offset": 9285785600, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.35.attn_output.weight", "offset": 9307289600, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.35.ffn_gate.weight", "offset": 9328793600, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.35.ffn_down.weight", "offset": 9386854400, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.35.ffn_up.weight", "offset": 9444915200, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 9502976000, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 9502996480, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.attn_q.weight", "offset": 9503016960, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.36.attn_k.weight", "offset": 9524520960, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.36.attn_v.weight", "offset": 9546024960, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.36.attn_output.weight", "offset": 9567528960, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.36.ffn_gate.weight", "offset": 9589032960, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.36.ffn_down.weight", "offset": 9647093760, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.36.ffn_up.weight", "offset": 9705154560, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.36.attn_norm.weight", "offset": 9763215360, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 9763235840, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.37.attn_q.weight", "offset": 9763256320, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.37.attn_k.weight", "offset": 9784760320, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.37.attn_v.weight", "offset": 9806264320, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.37.attn_output.weight", "offset": 9827768320, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.37.ffn_gate.weight", "offset": 9849272320, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.37.ffn_down.weight", "offset": 9907333120, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.37.ffn_up.weight", "offset": 9965393920, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.37.attn_norm.weight", "offset": 10023454720, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 10023475200, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.38.attn_q.weight", "offset": 10023495680, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.38.attn_k.weight", "offset": 10044999680, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.38.attn_v.weight", "offset": 10066503680, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.38.attn_output.weight", "offset": 10088007680, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.38.ffn_gate.weight", "offset": 10109511680, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.38.ffn_down.weight", "offset": 10167572480, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.38.ffn_up.weight", "offset": 10225633280, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 10283694080, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 10283714560, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.39.attn_q.weight", "offset": 10283735040, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.39.attn_k.weight", "offset": 10305239040, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.39.attn_v.weight", "offset": 10326743040, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.39.attn_output.weight", "offset": 10348247040, "shape": [ 5120, 5120 ], "size": 21504000, "type": 14 }, { "name": "blk.39.ffn_gate.weight", "offset": 10369751040, "shape": [ 5120, 13824 ], "size": 21504000, "type": 14 }, { "name": "blk.39.ffn_down.weight", "offset": 10427811840, "shape": [ 13824, 5120 ], "size": 58060800, "type": 14 }, { "name": "blk.39.ffn_up.weight", "offset": 10485872640, "shape": [ 5120, 13824 ], "size": 58060800, "type": 14 }, { "name": "blk.39.attn_norm.weight", "offset": 10543933440, "shape": [ 5120 ], "size": 58060800, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 10543953920, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "output_norm.weight", "offset": 10543974400, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "output.weight", "offset": 10543994880, "shape": [ 5120, 32000 ], "size": 20480, "type": 14 } ], "version": 2 }